vnet/vnet/dpo/lookup_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/lookup_dpo.h>
  18 #include <vnet/dpo/load_balance.h>
  19 #include <vnet/mpls/mpls.h>
  20 #include <vnet/fib/fib_table.h>
  21 #include <vnet/fib/ip4_fib.h>
  22 #include <vnet/fib/ip6_fib.h>
  23 #include <vnet/fib/mpls_fib.h>
  24
  25 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
  26
  27 /**
  28  * @brief Enumeration of the lookup subtypes
  29  */
  30 typedef enum lookup_sub_type_t_
  31 {
  32     LOOKUP_SUB_TYPE_SRC,
  33     LOOKUP_SUB_TYPE_DST,
  34     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
  35 } lookup_sub_type_t;
  36 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
  37
  38 #define FOR_EACH_LOOKUP_SUB_TYPE(_st)                                   \
  39     for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
  40
  41 /**
  42  * @brief pool of all MPLS Label DPOs
  43  */
  44 lookup_dpo_t *lookup_dpo_pool;
  45
  46 /**
  47  * @brief An array of registered DPO type values for the sub-types
  48  */
  49 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
  50
  51 static lookup_dpo_t *
  52 lookup_dpo_alloc (void)
  53 {
  54     lookup_dpo_t *lkd;
  55
  56     pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
  57
  58     return (lkd);
  59 }
  60
  61 static index_t
  62 lookup_dpo_get_index (lookup_dpo_t *lkd)
  63 {
  64     return (lkd - lookup_dpo_pool);
  65 }
  66
  67 static void
  68 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
  69                           dpo_proto_t proto,
  70                           lookup_input_t input,
  71                           lookup_table_t table_config,
  72                           dpo_id_t *dpo)
  73 {
  74     lookup_dpo_t *lkd;
  75     dpo_type_t type;
  76
  77     lkd = lookup_dpo_alloc();
  78     lkd->lkd_fib_index = fib_index;
  79     lkd->lkd_proto = proto;
  80     lkd->lkd_input = input;
  81     lkd->lkd_table = table_config;
  82
  83     /*
  84      * use the input type to select the lookup sub-type
  85      */
  86     type = 0;
  87
  88     switch (input)
  89     {
  90     case LOOKUP_INPUT_SRC_ADDR:
  91         type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
  92         break;
  93     case LOOKUP_INPUT_DST_ADDR:
  94         switch (table_config)
  95         {
  96         case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
  97             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
  98             break;
  99         case LOOKUP_TABLE_FROM_CONFIG:
 100             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
 101             break;
 102         }
 103     }
 104
 105     if (0 == type)
 106     {
 107         dpo_reset(dpo);
 108     }
 109     else
 110     {
 111         dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
 112     }
 113 }
 114
 115 void
 116 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
 117                                     dpo_proto_t proto,
 118                                     lookup_input_t input,
 119                                     lookup_table_t table_config,
 120                                     dpo_id_t *dpo)
 121 {
 122     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 123     {
 124         fib_table_lock(fib_index, dpo_proto_to_fib(proto));
 125     }
 126     lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
 127 }
 128
 129 void
 130 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 131                                    dpo_proto_t proto,
 132                                    lookup_input_t input,
 133                                    lookup_table_t table_config,
 134                                    dpo_id_t *dpo)
 135 {
 136     fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
 137
 138     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 139     {
 140         fib_index =
 141             fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 142                                               table_id);
 143     }
 144
 145     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
 146     lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
 147 }
 148
 149 u8*
 150 format_lookup_dpo (u8 *s, va_list *args)
 151 {
 152     index_t index = va_arg (*args, index_t);
 153     lookup_dpo_t *lkd;
 154
 155     lkd = lookup_dpo_get(index);
 156
 157     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
 158     {
 159         s = format(s, "%s lookup in interface's %U table",
 160                    lookup_input_names[lkd->lkd_input],
 161                    format_dpo_proto, lkd->lkd_proto);
 162     }
 163     else
 164     {
 165         s = format(s, "%s lookup in %U",
 166                    lookup_input_names[lkd->lkd_input],
 167                    format_fib_table_name, lkd->lkd_fib_index,
 168                    dpo_proto_to_fib(lkd->lkd_proto));
 169     }
 170     return (s);
 171 }
 172
 173 static void
 174 lookup_dpo_lock (dpo_id_t *dpo)
 175 {
 176     lookup_dpo_t *lkd;
 177
 178     lkd = lookup_dpo_get(dpo->dpoi_index);
 179
 180     lkd->lkd_locks++;
 181 }
 182
 183 static void
 184 lookup_dpo_unlock (dpo_id_t *dpo)
 185 {
 186     lookup_dpo_t *lkd;
 187
 188     lkd = lookup_dpo_get(dpo->dpoi_index);
 189
 190     lkd->lkd_locks--;
 191
 192     if (0 == lkd->lkd_locks)
 193     {
 194         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
 195         {
 196             fib_table_unlock(lkd->lkd_fib_index,
 197                              dpo_proto_to_fib(lkd->lkd_proto));
 198         }
 199         pool_put(lookup_dpo_pool, lkd);
 200     }
 201 }
 202
 203 always_inline void
 204 ip4_src_fib_lookup_one (u32 src_fib_index0,
 205                         const ip4_address_t * addr0,
 206                         u32 * src_adj_index0)
 207 {
 208     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 209     ip4_fib_mtrie_t * mtrie0;
 210
 211     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 212
 213     leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 214     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 0);
 215     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
 216     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 217     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 218
 219     /* Handle default route. */
 220     leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 221     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 222 }
 223
 224 always_inline void
 225 ip4_src_fib_lookup_two (u32 src_fib_index0,
 226                         u32 src_fib_index1,
 227                         const ip4_address_t * addr0,
 228                         const ip4_address_t * addr1,
 229                         u32 * src_adj_index0,
 230                         u32 * src_adj_index1)
 231 {
 232     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 233     ip4_fib_mtrie_t * mtrie0, * mtrie1;
 234
 235     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 236     mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
 237
 238     leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 239
 240     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 0);
 241     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 0);
 242
 243     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
 244     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 1);
 245
 246     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 247     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
 248
 249     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 250     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
 251
 252     /* Handle default route. */
 253     leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 254     leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
 255     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 256     src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 257 }
 258
 259 /**
 260  * @brief Lookup trace  data
 261  */
 262 typedef struct lookup_trace_t_
 263 {
 264     union {
 265         ip46_address_t addr;
 266         mpls_unicast_header_t hdr;
 267     };
 268     fib_node_index_t fib_index;
 269     index_t lbi;
 270 } lookup_trace_t;
 271
 272
 273 always_inline uword
 274 lookup_dpo_ip4_inline (vlib_main_t * vm,
 275                        vlib_node_runtime_t * node,
 276                        vlib_frame_t * from_frame,
 277                        int input_src_addr,
 278                        int table_from_interface)
 279 {
 280     u32 n_left_from, next_index, * from, * to_next;
 281     u32 cpu_index = os_get_cpu_number();
 282     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 283
 284     from = vlib_frame_vector_args (from_frame);
 285     n_left_from = from_frame->n_vectors;
 286
 287     next_index = node->cached_next_index;
 288
 289     while (n_left_from > 0)
 290     {
 291         u32 n_left_to_next;
 292
 293         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 294
 295         while (n_left_from >= 4 && n_left_to_next > 2)
 296         {
 297             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 298             flow_hash_config_t flow_hash_config0;
 299             const ip4_address_t *input_addr0;
 300             const load_balance_t *lb0;
 301             const lookup_dpo_t * lkd0;
 302             const ip4_header_t * ip0;
 303             const dpo_id_t *dpo0;
 304             vlib_buffer_t * b0;
 305             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 306             flow_hash_config_t flow_hash_config1;
 307             const ip4_address_t *input_addr1;
 308             const load_balance_t *lb1;
 309             const lookup_dpo_t * lkd1;
 310             const ip4_header_t * ip1;
 311             const dpo_id_t *dpo1;
 312             vlib_buffer_t * b1;
 313
 314             /* Prefetch next iteration. */
 315             {
 316                 vlib_buffer_t * p2, * p3;
 317
 318                 p2 = vlib_get_buffer (vm, from[2]);
 319                 p3 = vlib_get_buffer (vm, from[3]);
 320
 321                 vlib_prefetch_buffer_header (p2, LOAD);
 322                 vlib_prefetch_buffer_header (p3, LOAD);
 323
 324                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 325                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 326             }
 327
 328             bi0 = from[0];
 329             to_next[0] = bi0;
 330             bi1 = from[1];
 331             to_next[1] = bi1;
 332             from += 2;
 333             to_next += 2;
 334             n_left_from -= 2;
 335             n_left_to_next -= 2;
 336
 337             b0 = vlib_get_buffer (vm, bi0);
 338             ip0 = vlib_buffer_get_current (b0);
 339             b1 = vlib_get_buffer (vm, bi1);
 340             ip1 = vlib_buffer_get_current (b1);
 341
 342             /* dst lookup was done by ip4 lookup */
 343             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 344             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 345             lkd0 = lookup_dpo_get(lkdi0);
 346             lkd1 = lookup_dpo_get(lkdi1);
 347
 348             /*
 349              * choose between a lookup using the fib index in the DPO
 350              * or getting the FIB index from the interface.
 351              */
 352             if (table_from_interface)
 353             {
 354                 fib_index0 =
 355                     ip4_fib_table_get_index_for_sw_if_index(
 356                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 357                 fib_index1 =
 358                     ip4_fib_table_get_index_for_sw_if_index(
 359                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 360             }
 361             else
 362             {
 363                 fib_index0 = lkd0->lkd_fib_index;
 364                 fib_index1 = lkd1->lkd_fib_index;
 365             }
 366
 367             /*
 368              * choose between a source or destination address lookup in the table
 369              */
 370             if (input_src_addr)
 371             {
 372                 input_addr0 = &ip0->src_address;
 373                 input_addr1 = &ip1->src_address;
 374             }
 375             else
 376             {
 377                 input_addr0 = &ip0->dst_address;
 378                 input_addr1 = &ip1->dst_address;
 379             }
 380
 381             /* do lookup */
 382             ip4_src_fib_lookup_one (fib_index0, input_addr0, &lbi0);
 383             ip4_src_fib_lookup_one (fib_index1, input_addr1, &lbi1);
 384             lb0 = load_balance_get(lbi0);
 385             lb1 = load_balance_get(lbi1);
 386
 387             /* Use flow hash to compute multipath adjacency. */
 388             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 389             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 390
 391             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 392             {
 393                 flow_hash_config0 = lb0->lb_hash_config;
 394                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 395                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 396             }
 397
 398             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 399             {
 400                 flow_hash_config1 = lb1->lb_hash_config;
 401                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 402                     ip4_compute_flow_hash (ip1, flow_hash_config1);
 403             }
 404
 405             dpo0 = load_balance_get_bucket_i(lb0,
 406                                              (hash_c0 &
 407                                               (lb0->lb_n_buckets_minus_1)));
 408             dpo1 = load_balance_get_bucket_i(lb1,
 409                                              (hash_c1 &
 410                                               (lb1->lb_n_buckets_minus_1)));
 411
 412             next0 = dpo0->dpoi_next_node;
 413             next1 = dpo1->dpoi_next_node;
 414             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 415             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 416
 417             vlib_increment_combined_counter
 418                 (cm, cpu_index, lbi0, 1,
 419                  vlib_buffer_length_in_chain (vm, b0));
 420             vlib_increment_combined_counter
 421                 (cm, cpu_index, lbi1, 1,
 422                  vlib_buffer_length_in_chain (vm, b1));
 423
 424             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 425             {
 426                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 427                                                      b0, sizeof (*tr));
 428                 tr->fib_index = fib_index0;
 429                 tr->lbi = lbi0;
 430                 tr->addr.ip4 = *input_addr0;
 431             }
 432             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 433             {
 434                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 435                                                      b1, sizeof (*tr));
 436                 tr->fib_index = fib_index1;
 437                 tr->lbi = lbi1;
 438                 tr->addr.ip4 = *input_addr1;
 439             }
 440
 441             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 442                                              to_next, n_left_to_next,
 443                                              bi0, bi1, next0, next1);
 444         }
 445
 446         while (n_left_from > 0 && n_left_to_next > 0)
 447         {
 448             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 449             flow_hash_config_t flow_hash_config0;
 450             const ip4_address_t *input_addr;
 451             const load_balance_t *lb0;
 452             const lookup_dpo_t * lkd0;
 453             const ip4_header_t * ip0;
 454             const dpo_id_t *dpo0;
 455             vlib_buffer_t * b0;
 456
 457             bi0 = from[0];
 458             to_next[0] = bi0;
 459             from += 1;
 460             to_next += 1;
 461             n_left_from -= 1;
 462             n_left_to_next -= 1;
 463
 464             b0 = vlib_get_buffer (vm, bi0);
 465             ip0 = vlib_buffer_get_current (b0);
 466
 467             /* dst lookup was done by ip4 lookup */
 468             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 469             lkd0 = lookup_dpo_get(lkdi0);
 470
 471             /*
 472              * choose between a lookup using the fib index in the DPO
 473              * or getting the FIB index from the interface.
 474              */
 475             if (table_from_interface)
 476             {
 477                 fib_index0 =
 478                     ip4_fib_table_get_index_for_sw_if_index(
 479                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 480             }
 481             else
 482             {
 483                 fib_index0 = lkd0->lkd_fib_index;
 484             }
 485
 486             /*
 487              * choose between a source or destination address lookup in the table
 488              */
 489             if (input_src_addr)
 490             {
 491                 input_addr = &ip0->src_address;
 492             }
 493             else
 494             {
 495                 input_addr = &ip0->dst_address;
 496             }
 497
 498             /* do lookup */
 499             ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
 500             lb0 = load_balance_get(lbi0);
 501
 502             /* Use flow hash to compute multipath adjacency. */
 503             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 504
 505             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 506             {
 507                 flow_hash_config0 = lb0->lb_hash_config;
 508                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 509                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 510             }
 511
 512             dpo0 = load_balance_get_bucket_i(lb0,
 513                                              (hash_c0 &
 514                                               (lb0->lb_n_buckets_minus_1)));
 515
 516             next0 = dpo0->dpoi_next_node;
 517             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 518
 519             vlib_increment_combined_counter
 520                 (cm, cpu_index, lbi0, 1,
 521                  vlib_buffer_length_in_chain (vm, b0));
 522
 523             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 524             {
 525                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 526                                                      b0, sizeof (*tr));
 527                 tr->fib_index = fib_index0;
 528                 tr->lbi = lbi0;
 529                 tr->addr.ip4 = *input_addr;
 530             }
 531
 532             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 533                                             n_left_to_next, bi0, next0);
 534         }
 535         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 536     }
 537     return from_frame->n_vectors;
 538 }
 539
 540 static u8 *
 541 format_lookup_trace (u8 * s, va_list * args)
 542 {
 543     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 544     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 545     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 546     uword indent = format_get_indent (s);
 547     s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
 548                 format_white_space, indent,
 549                 t->fib_index,
 550                 format_ip46_address, &t->addr, IP46_TYPE_ANY,
 551                 t->lbi);
 552     return s;
 553 }
 554
 555 always_inline uword
 556 lookup_ip4_dst (vlib_main_t * vm,
 557                 vlib_node_runtime_t * node,
 558                 vlib_frame_t * from_frame)
 559 {
 560     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
 561 }
 562
 563 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
 564     .function = lookup_ip4_dst,
 565     .name = "lookup-ip4-dst",
 566     .vector_size = sizeof (u32),
 567     .sibling_of = "ip4-lookup",
 568     .format_trace = format_lookup_trace,
 569 };
 570 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_node, lookup_ip4_dst)
 571
 572 always_inline uword
 573 lookup_ip4_dst_itf (vlib_main_t * vm,
 574                     vlib_node_runtime_t * node,
 575                     vlib_frame_t * from_frame)
 576 {
 577     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
 578 }
 579
 580 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
 581     .function = lookup_ip4_dst_itf,
 582     .name = "lookup-ip4-dst-itf",
 583     .vector_size = sizeof (u32),
 584     .sibling_of = "ip4-lookup",
 585     .format_trace = format_lookup_trace,
 586 };
 587 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_itf_node, lookup_ip4_dst_itf)
 588
 589 always_inline uword
 590 lookup_ip4_src (vlib_main_t * vm,
 591                 vlib_node_runtime_t * node,
 592                 vlib_frame_t * from_frame)
 593 {
 594     return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
 595 }
 596
 597 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
 598     .function = lookup_ip4_src,
 599     .name = "lookup-ip4-src",
 600     .vector_size = sizeof (u32),
 601     .format_trace = format_lookup_trace,
 602     .sibling_of = "ip4-lookup",
 603 };
 604 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_src_node, lookup_ip4_src)
 605
 606 always_inline uword
 607 lookup_dpo_ip6_inline (vlib_main_t * vm,
 608                        vlib_node_runtime_t * node,
 609                        vlib_frame_t * from_frame,
 610                        int input_src_addr,
 611                        int table_from_interface)
 612 {
 613     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 614     u32 n_left_from, next_index, * from, * to_next;
 615     u32 cpu_index = os_get_cpu_number();
 616
 617     from = vlib_frame_vector_args (from_frame);
 618     n_left_from = from_frame->n_vectors;
 619
 620     next_index = node->cached_next_index;
 621
 622     while (n_left_from > 0)
 623     {
 624         u32 n_left_to_next;
 625
 626         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 627
 628         while (n_left_from >= 4 && n_left_to_next > 2)
 629         {
 630             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 631             flow_hash_config_t flow_hash_config0;
 632             const ip6_address_t *input_addr0;
 633             const load_balance_t *lb0;
 634             const lookup_dpo_t * lkd0;
 635             const ip6_header_t * ip0;
 636             const dpo_id_t *dpo0;
 637             vlib_buffer_t * b0;
 638             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 639             flow_hash_config_t flow_hash_config1;
 640             const ip6_address_t *input_addr1;
 641             const load_balance_t *lb1;
 642             const lookup_dpo_t * lkd1;
 643             const ip6_header_t * ip1;
 644             const dpo_id_t *dpo1;
 645             vlib_buffer_t * b1;
 646
 647             /* Prefetch next iteration. */
 648             {
 649                 vlib_buffer_t * p2, * p3;
 650
 651                 p2 = vlib_get_buffer (vm, from[2]);
 652                 p3 = vlib_get_buffer (vm, from[3]);
 653
 654                 vlib_prefetch_buffer_header (p2, LOAD);
 655                 vlib_prefetch_buffer_header (p3, LOAD);
 656
 657                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 658                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 659             }
 660
 661             bi0 = from[0];
 662             to_next[0] = bi0;
 663             bi1 = from[1];
 664             to_next[1] = bi1;
 665             from += 2;
 666             to_next += 2;
 667             n_left_from -= 2;
 668             n_left_to_next -= 2;
 669
 670             b0 = vlib_get_buffer (vm, bi0);
 671             ip0 = vlib_buffer_get_current (b0);
 672             b1 = vlib_get_buffer (vm, bi1);
 673             ip1 = vlib_buffer_get_current (b1);
 674
 675             /* dst lookup was done by ip6 lookup */
 676             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 677             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 678             lkd0 = lookup_dpo_get(lkdi0);
 679             lkd1 = lookup_dpo_get(lkdi1);
 680
 681             /*
 682              * choose between a lookup using the fib index in the DPO
 683              * or getting the FIB index from the interface.
 684              */
 685             if (table_from_interface)
 686             {
 687                 fib_index0 =
 688                     ip6_fib_table_get_index_for_sw_if_index(
 689                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 690                 fib_index1 =
 691                     ip6_fib_table_get_index_for_sw_if_index(
 692                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 693             }
 694             else
 695             {
 696                 fib_index0 = lkd0->lkd_fib_index;
 697                 fib_index1 = lkd1->lkd_fib_index;
 698             }
 699
 700             /*
 701              * choose between a source or destination address lookup in the table
 702              */
 703             if (input_src_addr)
 704             {
 705                 input_addr0 = &ip0->src_address;
 706                 input_addr1 = &ip1->src_address;
 707             }
 708             else
 709             {
 710                 input_addr0 = &ip0->dst_address;
 711                 input_addr1 = &ip1->dst_address;
 712             }
 713
 714             /* do src lookup */
 715             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 716                                                fib_index0,
 717                                                input_addr0);
 718             lbi1 = ip6_fib_table_fwding_lookup(&ip6_main,
 719                                                fib_index1,
 720                                                input_addr1);
 721             lb0 = load_balance_get(lbi0);
 722             lb1 = load_balance_get(lbi1);
 723
 724             /* Use flow hash to compute multipath adjacency. */
 725             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 726             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 727
 728             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 729             {
 730                 flow_hash_config0 = lb0->lb_hash_config;
 731                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 732                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 733             }
 734
 735             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 736             {
 737                 flow_hash_config1 = lb1->lb_hash_config;
 738                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 739                     ip6_compute_flow_hash (ip1, flow_hash_config1);
 740             }
 741
 742             dpo0 = load_balance_get_bucket_i(lb0,
 743                                              (hash_c0 &
 744                                               (lb0->lb_n_buckets_minus_1)));
 745             dpo1 = load_balance_get_bucket_i(lb1,
 746                                              (hash_c1 &
 747                                               (lb1->lb_n_buckets_minus_1)));
 748
 749             next0 = dpo0->dpoi_next_node;
 750             next1 = dpo1->dpoi_next_node;
 751             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 752             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 753
 754             vlib_increment_combined_counter
 755                 (cm, cpu_index, lbi0, 1,
 756                  vlib_buffer_length_in_chain (vm, b0));
 757             vlib_increment_combined_counter
 758                 (cm, cpu_index, lbi1, 1,
 759                  vlib_buffer_length_in_chain (vm, b1));
 760
 761             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 762             {
 763                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 764                                                      b0, sizeof (*tr));
 765                 tr->fib_index = fib_index0;
 766                 tr->lbi = lbi0;
 767                 tr->addr.ip6 = *input_addr0;
 768             }
 769             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 770             {
 771                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 772                                                      b1, sizeof (*tr));
 773                 tr->fib_index = fib_index1;
 774                 tr->lbi = lbi1;
 775                 tr->addr.ip6 = *input_addr1;
 776             }
 777             vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
 778                                             n_left_to_next, bi0, bi1,
 779                                             next0, next1);
 780         }
 781         while (n_left_from > 0 && n_left_to_next > 0)
 782         {
 783             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 784             flow_hash_config_t flow_hash_config0;
 785             const ip6_address_t *input_addr0;
 786             const load_balance_t *lb0;
 787             const lookup_dpo_t * lkd0;
 788             const ip6_header_t * ip0;
 789             const dpo_id_t *dpo0;
 790             vlib_buffer_t * b0;
 791
 792             bi0 = from[0];
 793             to_next[0] = bi0;
 794             from += 1;
 795             to_next += 1;
 796             n_left_from -= 1;
 797             n_left_to_next -= 1;
 798
 799             b0 = vlib_get_buffer (vm, bi0);
 800             ip0 = vlib_buffer_get_current (b0);
 801
 802             /* dst lookup was done by ip6 lookup */
 803             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 804             lkd0 = lookup_dpo_get(lkdi0);
 805
 806             /*
 807              * choose between a lookup using the fib index in the DPO
 808              * or getting the FIB index from the interface.
 809              */
 810             if (table_from_interface)
 811             {
 812                 fib_index0 =
 813                     ip6_fib_table_get_index_for_sw_if_index(
 814                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 815             }
 816             else
 817             {
 818                 fib_index0 = lkd0->lkd_fib_index;
 819             }
 820
 821             /*
 822              * choose between a source or destination address lookup in the table
 823              */
 824             if (input_src_addr)
 825             {
 826                 input_addr0 = &ip0->src_address;
 827             }
 828             else
 829             {
 830                 input_addr0 = &ip0->dst_address;
 831             }
 832
 833             /* do src lookup */
 834             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 835                                                fib_index0,
 836                                                input_addr0);
 837             lb0 = load_balance_get(lbi0);
 838
 839             /* Use flow hash to compute multipath adjacency. */
 840             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 841
 842             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 843             {
 844                 flow_hash_config0 = lb0->lb_hash_config;
 845                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 846                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 847             }
 848
 849             dpo0 = load_balance_get_bucket_i(lb0,
 850                                              (hash_c0 &
 851                                               (lb0->lb_n_buckets_minus_1)));
 852
 853             next0 = dpo0->dpoi_next_node;
 854             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 855
 856             vlib_increment_combined_counter
 857                 (cm, cpu_index, lbi0, 1,
 858                  vlib_buffer_length_in_chain (vm, b0));
 859
 860             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 861             {
 862                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 863                                                      b0, sizeof (*tr));
 864                 tr->fib_index = fib_index0;
 865                 tr->lbi = lbi0;
 866                 tr->addr.ip6 = *input_addr0;
 867             }
 868             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 869                                             n_left_to_next, bi0, next0);
 870         }
 871         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 872     }
 873     return from_frame->n_vectors;
 874 }
 875
 876 always_inline uword
 877 lookup_ip6_dst (vlib_main_t * vm,
 878                 vlib_node_runtime_t * node,
 879                 vlib_frame_t * from_frame)
 880 {
 881     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 0));
 882 }
 883
 884 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
 885     .function = lookup_ip6_dst,
 886     .name = "lookup-ip6-dst",
 887     .vector_size = sizeof (u32),
 888     .format_trace = format_lookup_trace,
 889     .sibling_of = "ip6-lookup",
 890 };
 891 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_node, lookup_ip6_dst)
 892
 893 always_inline uword
 894 lookup_ip6_dst_itf (vlib_main_t * vm,
 895                     vlib_node_runtime_t * node,
 896                     vlib_frame_t * from_frame)
 897 {
 898     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 1));
 899 }
 900
 901 VLIB_REGISTER_NODE (lookup_ip6_dst_itf_node) = {
 902     .function = lookup_ip6_dst_itf,
 903     .name = "lookup-ip6-dst-itf",
 904     .vector_size = sizeof (u32),
 905     .format_trace = format_lookup_trace,
 906     .sibling_of = "ip6-lookup",
 907 };
 908 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_itf_node, lookup_ip6_dst_itf)
 909
 910 always_inline uword
 911 lookup_ip6_src (vlib_main_t * vm,
 912                 vlib_node_runtime_t * node,
 913                 vlib_frame_t * from_frame)
 914 {
 915     return (lookup_dpo_ip6_inline(vm, node, from_frame, 1, 0));
 916 }
 917
 918 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
 919     .function = lookup_ip6_src,
 920     .name = "lookup-ip6-src",
 921     .vector_size = sizeof (u32),
 922     .format_trace = format_lookup_trace,
 923     .sibling_of = "ip6-lookup",
 924 };
 925 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_src_node, lookup_ip6_src)
 926
 927 always_inline uword
 928 lookup_dpo_mpls_inline (vlib_main_t * vm,
 929                        vlib_node_runtime_t * node,
 930                        vlib_frame_t * from_frame,
 931                        int table_from_interface)
 932 {
 933     u32 n_left_from, next_index, * from, * to_next;
 934     u32 cpu_index = os_get_cpu_number();
 935     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 936
 937     from = vlib_frame_vector_args (from_frame);
 938     n_left_from = from_frame->n_vectors;
 939
 940     next_index = node->cached_next_index;
 941
 942     while (n_left_from > 0)
 943     {
 944         u32 n_left_to_next;
 945
 946         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 947
 948         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
 949         /*   } */
 950
 951         while (n_left_from > 0 && n_left_to_next > 0)
 952         {
 953             u32 bi0, lkdi0, lbi0, fib_index0,  next0;
 954             const mpls_unicast_header_t * hdr0;
 955             const load_balance_t *lb0;
 956             const lookup_dpo_t * lkd0;
 957             const dpo_id_t *dpo0;
 958             vlib_buffer_t * b0;
 959
 960             bi0 = from[0];
 961             to_next[0] = bi0;
 962             from += 1;
 963             to_next += 1;
 964             n_left_from -= 1;
 965             n_left_to_next -= 1;
 966
 967             b0 = vlib_get_buffer (vm, bi0);
 968             hdr0 = vlib_buffer_get_current (b0);
 969
 970             /* dst lookup was done by mpls lookup */
 971             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 972             lkd0 = lookup_dpo_get(lkdi0);
 973
 974             /*
 975              * choose between a lookup using the fib index in the DPO
 976              * or getting the FIB index from the interface.
 977              */
 978             if (table_from_interface)
 979             {
 980                 fib_index0 =
 981                     mpls_fib_table_get_index_for_sw_if_index(
 982                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 983             }
 984             else
 985             {
 986                 fib_index0 = lkd0->lkd_fib_index;
 987             }
 988
 989             /* do lookup */
 990             lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
 991             lb0  = load_balance_get(lbi0);
 992             dpo0 = load_balance_get_bucket_i(lb0, 0);
 993
 994             next0 = dpo0->dpoi_next_node;
 995             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 996
 997             vlib_increment_combined_counter
 998                 (cm, cpu_index, lbi0, 1,
 999                  vlib_buffer_length_in_chain (vm, b0));
1000
1001             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1002             {
1003                 lookup_trace_t *tr = vlib_add_trace (vm, node,
1004                                                      b0, sizeof (*tr));
1005                 tr->fib_index = fib_index0;
1006                 tr->lbi = lbi0;
1007                 tr->hdr = *hdr0;
1008             }
1009
1010            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1011                                             n_left_to_next, bi0, next0);
1012         }
1013         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1014     }
1015     return from_frame->n_vectors;
1016 }
1017
1018 static u8 *
1019 format_lookup_mpls_trace (u8 * s, va_list * args)
1020 {
1021     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1022     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1023     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
1024     uword indent = format_get_indent (s);
1025     mpls_unicast_header_t hdr;
1026
1027     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
1028
1029     s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
1030                 format_white_space, indent,
1031                 t->fib_index,
1032                 format_mpls_header, hdr,
1033                 t->lbi);
1034     return s;
1035 }
1036
1037 always_inline uword
1038 lookup_mpls_dst (vlib_main_t * vm,
1039                 vlib_node_runtime_t * node,
1040                 vlib_frame_t * from_frame)
1041 {
1042     return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
1043 }
1044
1045 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
1046     .function = lookup_mpls_dst,
1047     .name = "lookup-mpls-dst",
1048     .vector_size = sizeof (u32),
1049     .sibling_of = "mpls-lookup",
1050     .format_trace = format_lookup_mpls_trace,
1051     .n_next_nodes = 0,
1052 };
1053 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_node, lookup_mpls_dst)
1054
1055 always_inline uword
1056 lookup_mpls_dst_itf (vlib_main_t * vm,
1057                     vlib_node_runtime_t * node,
1058                     vlib_frame_t * from_frame)
1059 {
1060     return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
1061 }
1062
1063 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
1064     .function = lookup_mpls_dst_itf,
1065     .name = "lookup-mpls-dst-itf",
1066     .vector_size = sizeof (u32),
1067     .sibling_of = "mpls-lookup",
1068     .format_trace = format_lookup_mpls_trace,
1069     .n_next_nodes = 0,
1070 };
1071 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf)
1072
1073 static void
1074 lookup_dpo_mem_show (void)
1075 {
1076     fib_show_memory_usage("Lookup",
1077                           pool_elts(lookup_dpo_pool),
1078                           pool_len(lookup_dpo_pool),
1079                           sizeof(lookup_dpo_t));
1080 }
1081
1082 const static dpo_vft_t lkd_vft = {
1083     .dv_lock = lookup_dpo_lock,
1084     .dv_unlock = lookup_dpo_unlock,
1085     .dv_format = format_lookup_dpo,
1086 };
1087 const static dpo_vft_t lkd_vft_w_mem_show = {
1088     .dv_lock = lookup_dpo_lock,
1089     .dv_unlock = lookup_dpo_unlock,
1090     .dv_format = format_lookup_dpo,
1091     .dv_mem_show = lookup_dpo_mem_show,
1092 };
1093
1094 const static char* const lookup_src_ip4_nodes[] =
1095 {
1096     "lookup-ip4-src",
1097     NULL,
1098 };
1099 const static char* const lookup_src_ip6_nodes[] =
1100 {
1101     "lookup-ip6-src",
1102     NULL,
1103 };
1104 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
1105 {
1106     [DPO_PROTO_IP4]  = lookup_src_ip4_nodes,
1107     [DPO_PROTO_IP6]  = lookup_src_ip6_nodes,
1108     [DPO_PROTO_MPLS] = NULL,
1109 };
1110
1111 const static char* const lookup_dst_ip4_nodes[] =
1112 {
1113     "lookup-ip4-dst",
1114     NULL,
1115 };
1116 const static char* const lookup_dst_ip6_nodes[] =
1117 {
1118     "lookup-ip6-dst",
1119     NULL,
1120 };
1121 const static char* const lookup_dst_mpls_nodes[] =
1122 {
1123     "lookup-mpls-dst",
1124     NULL,
1125 };
1126 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
1127 {
1128     [DPO_PROTO_IP4]  = lookup_dst_ip4_nodes,
1129     [DPO_PROTO_IP6]  = lookup_dst_ip6_nodes,
1130     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
1131 };
1132
1133 const static char* const lookup_dst_from_interface_ip4_nodes[] =
1134 {
1135     "lookup-ip4-dst-itf",
1136     NULL,
1137 };
1138 const static char* const lookup_dst_from_interface_ip6_nodes[] =
1139 {
1140     "lookup-ip6-dst-itf",
1141     NULL,
1142 };
1143 const static char* const lookup_dst_from_interface_mpls_nodes[] =
1144 {
1145     "lookup-mpls-dst-itf",
1146     NULL,
1147 };
1148 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
1149 {
1150     [DPO_PROTO_IP4]  = lookup_dst_from_interface_ip4_nodes,
1151     [DPO_PROTO_IP6]  = lookup_dst_from_interface_ip6_nodes,
1152     [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
1153 };
1154
1155
1156 void
1157 lookup_dpo_module_init (void)
1158 {
1159     dpo_register(DPO_LOOKUP, &lkd_vft_w_mem_show, NULL);
1160
1161     /*
1162      * There are various sorts of lookup; src or dst addr v4 /v6 etc.
1163      * there isn't an object type for each (there is only the lookup_dpo_t),
1164      * but, for performance reasons, there is a data plane function, and hence
1165      * VLIB node for each. VLIB graph node construction is based on DPO types
1166      * so we create sub-types.
1167      */
1168     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
1169         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
1170     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
1171         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
1172     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
1173         dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes);
1174 }