src/vnet/dpo/lookup_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/lookup_dpo.h>
  18 #include <vnet/dpo/load_balance.h>
  19 #include <vnet/mpls/mpls.h>
  20 #include <vnet/fib/fib_table.h>
  21 #include <vnet/fib/ip4_fib.h>
  22 #include <vnet/fib/ip6_fib.h>
  23 #include <vnet/fib/mpls_fib.h>
  24
  25 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
  26
  27 /**
  28  * @brief Enumeration of the lookup subtypes
  29  */
  30 typedef enum lookup_sub_type_t_
  31 {
  32     LOOKUP_SUB_TYPE_SRC,
  33     LOOKUP_SUB_TYPE_DST,
  34     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
  35 } lookup_sub_type_t;
  36 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
  37
  38 #define FOR_EACH_LOOKUP_SUB_TYPE(_st)                                   \
  39     for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
  40
  41 /**
  42  * @brief pool of all MPLS Label DPOs
  43  */
  44 lookup_dpo_t *lookup_dpo_pool;
  45
  46 /**
  47  * @brief An array of registered DPO type values for the sub-types
  48  */
  49 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
  50
  51 static lookup_dpo_t *
  52 lookup_dpo_alloc (void)
  53 {
  54     lookup_dpo_t *lkd;
  55
  56     pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
  57
  58     return (lkd);
  59 }
  60
  61 static index_t
  62 lookup_dpo_get_index (lookup_dpo_t *lkd)
  63 {
  64     return (lkd - lookup_dpo_pool);
  65 }
  66
  67 static void
  68 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
  69                           dpo_proto_t proto,
  70                           lookup_input_t input,
  71                           lookup_table_t table_config,
  72                           dpo_id_t *dpo)
  73 {
  74     lookup_dpo_t *lkd;
  75     dpo_type_t type;
  76
  77     lkd = lookup_dpo_alloc();
  78     lkd->lkd_fib_index = fib_index;
  79     lkd->lkd_proto = proto;
  80     lkd->lkd_input = input;
  81     lkd->lkd_table = table_config;
  82
  83     /*
  84      * use the input type to select the lookup sub-type
  85      */
  86     type = 0;
  87
  88     switch (input)
  89     {
  90     case LOOKUP_INPUT_SRC_ADDR:
  91         type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
  92         break;
  93     case LOOKUP_INPUT_DST_ADDR:
  94         switch (table_config)
  95         {
  96         case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
  97             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
  98             break;
  99         case LOOKUP_TABLE_FROM_CONFIG:
 100             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
 101             break;
 102         }
 103     }
 104
 105     if (0 == type)
 106     {
 107         dpo_reset(dpo);
 108     }
 109     else
 110     {
 111         dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
 112     }
 113 }
 114
 115 void
 116 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
 117                                     dpo_proto_t proto,
 118                                     lookup_input_t input,
 119                                     lookup_table_t table_config,
 120                                     dpo_id_t *dpo)
 121 {
 122     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 123     {
 124         fib_table_lock(fib_index, dpo_proto_to_fib(proto));
 125     }
 126     lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
 127 }
 128
 129 void
 130 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 131                                    dpo_proto_t proto,
 132                                    lookup_input_t input,
 133                                    lookup_table_t table_config,
 134                                    dpo_id_t *dpo)
 135 {
 136     fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
 137
 138     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 139     {
 140         fib_index =
 141             fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 142                                               table_id);
 143     }
 144
 145     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
 146     lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
 147 }
 148
 149 u8*
 150 format_lookup_dpo (u8 *s, va_list *args)
 151 {
 152     index_t index = va_arg (*args, index_t);
 153     lookup_dpo_t *lkd;
 154
 155     lkd = lookup_dpo_get(index);
 156
 157     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
 158     {
 159         s = format(s, "%s lookup in interface's %U table",
 160                    lookup_input_names[lkd->lkd_input],
 161                    format_dpo_proto, lkd->lkd_proto);
 162     }
 163     else
 164     {
 165         s = format(s, "%s lookup in %U",
 166                    lookup_input_names[lkd->lkd_input],
 167                    format_fib_table_name, lkd->lkd_fib_index,
 168                    dpo_proto_to_fib(lkd->lkd_proto));
 169     }
 170     return (s);
 171 }
 172
 173 static void
 174 lookup_dpo_lock (dpo_id_t *dpo)
 175 {
 176     lookup_dpo_t *lkd;
 177
 178     lkd = lookup_dpo_get(dpo->dpoi_index);
 179
 180     lkd->lkd_locks++;
 181 }
 182
 183 static void
 184 lookup_dpo_unlock (dpo_id_t *dpo)
 185 {
 186     lookup_dpo_t *lkd;
 187
 188     lkd = lookup_dpo_get(dpo->dpoi_index);
 189
 190     lkd->lkd_locks--;
 191
 192     if (0 == lkd->lkd_locks)
 193     {
 194         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
 195         {
 196             fib_table_unlock(lkd->lkd_fib_index,
 197                              dpo_proto_to_fib(lkd->lkd_proto));
 198         }
 199         pool_put(lookup_dpo_pool, lkd);
 200     }
 201 }
 202
 203 always_inline void
 204 ip4_src_fib_lookup_one (u32 src_fib_index0,
 205                         const ip4_address_t * addr0,
 206                         u32 * src_adj_index0)
 207 {
 208     ip4_fib_mtrie_leaf_t leaf0;
 209     ip4_fib_mtrie_t * mtrie0;
 210
 211     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 212
 213     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 214     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
 215     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 216     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 217
 218     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 219 }
 220
 221 always_inline void
 222 ip4_src_fib_lookup_two (u32 src_fib_index0,
 223                         u32 src_fib_index1,
 224                         const ip4_address_t * addr0,
 225                         const ip4_address_t * addr1,
 226                         u32 * src_adj_index0,
 227                         u32 * src_adj_index1)
 228 {
 229     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 230     ip4_fib_mtrie_t * mtrie0, * mtrie1;
 231
 232     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 233     mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
 234
 235     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 236     leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, addr1);
 237
 238     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
 239     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 1);
 240
 241     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 242     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
 243
 244     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 245     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
 246
 247     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 248     src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 249 }
 250
 251 /**
 252  * @brief Lookup trace  data
 253  */
 254 typedef struct lookup_trace_t_
 255 {
 256     union {
 257         ip46_address_t addr;
 258         mpls_unicast_header_t hdr;
 259     };
 260     fib_node_index_t fib_index;
 261     index_t lbi;
 262 } lookup_trace_t;
 263
 264
 265 always_inline uword
 266 lookup_dpo_ip4_inline (vlib_main_t * vm,
 267                        vlib_node_runtime_t * node,
 268                        vlib_frame_t * from_frame,
 269                        int input_src_addr,
 270                        int table_from_interface)
 271 {
 272     u32 n_left_from, next_index, * from, * to_next;
 273     u32 cpu_index = os_get_cpu_number();
 274     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 275
 276     from = vlib_frame_vector_args (from_frame);
 277     n_left_from = from_frame->n_vectors;
 278
 279     next_index = node->cached_next_index;
 280
 281     while (n_left_from > 0)
 282     {
 283         u32 n_left_to_next;
 284
 285         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 286
 287         while (n_left_from >= 4 && n_left_to_next > 2)
 288         {
 289             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 290             flow_hash_config_t flow_hash_config0;
 291             const ip4_address_t *input_addr0;
 292             const load_balance_t *lb0;
 293             const lookup_dpo_t * lkd0;
 294             const ip4_header_t * ip0;
 295             const dpo_id_t *dpo0;
 296             vlib_buffer_t * b0;
 297             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 298             flow_hash_config_t flow_hash_config1;
 299             const ip4_address_t *input_addr1;
 300             const load_balance_t *lb1;
 301             const lookup_dpo_t * lkd1;
 302             const ip4_header_t * ip1;
 303             const dpo_id_t *dpo1;
 304             vlib_buffer_t * b1;
 305
 306             /* Prefetch next iteration. */
 307             {
 308                 vlib_buffer_t * p2, * p3;
 309
 310                 p2 = vlib_get_buffer (vm, from[2]);
 311                 p3 = vlib_get_buffer (vm, from[3]);
 312
 313                 vlib_prefetch_buffer_header (p2, LOAD);
 314                 vlib_prefetch_buffer_header (p3, LOAD);
 315
 316                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 317                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 318             }
 319
 320             bi0 = from[0];
 321             to_next[0] = bi0;
 322             bi1 = from[1];
 323             to_next[1] = bi1;
 324             from += 2;
 325             to_next += 2;
 326             n_left_from -= 2;
 327             n_left_to_next -= 2;
 328
 329             b0 = vlib_get_buffer (vm, bi0);
 330             ip0 = vlib_buffer_get_current (b0);
 331             b1 = vlib_get_buffer (vm, bi1);
 332             ip1 = vlib_buffer_get_current (b1);
 333
 334             /* dst lookup was done by ip4 lookup */
 335             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 336             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 337             lkd0 = lookup_dpo_get(lkdi0);
 338             lkd1 = lookup_dpo_get(lkdi1);
 339
 340             /*
 341              * choose between a lookup using the fib index in the DPO
 342              * or getting the FIB index from the interface.
 343              */
 344             if (table_from_interface)
 345             {
 346                 fib_index0 =
 347                     ip4_fib_table_get_index_for_sw_if_index(
 348                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 349                 fib_index1 =
 350                     ip4_fib_table_get_index_for_sw_if_index(
 351                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 352             }
 353             else
 354             {
 355                 fib_index0 = lkd0->lkd_fib_index;
 356                 fib_index1 = lkd1->lkd_fib_index;
 357             }
 358
 359             /*
 360              * choose between a source or destination address lookup in the table
 361              */
 362             if (input_src_addr)
 363             {
 364                 input_addr0 = &ip0->src_address;
 365                 input_addr1 = &ip1->src_address;
 366             }
 367             else
 368             {
 369                 input_addr0 = &ip0->dst_address;
 370                 input_addr1 = &ip1->dst_address;
 371             }
 372
 373             /* do lookup */
 374             ip4_src_fib_lookup_two (fib_index0, fib_index1,
 375                                     input_addr0, input_addr1,
 376                                     &lbi0, &lbi1);
 377             lb0 = load_balance_get(lbi0);
 378             lb1 = load_balance_get(lbi1);
 379
 380             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 381             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 382
 383             /* Use flow hash to compute multipath adjacency. */
 384             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 385             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 386
 387             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 388             {
 389                 flow_hash_config0 = lb0->lb_hash_config;
 390                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 391                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 392             }
 393
 394             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 395             {
 396                 flow_hash_config1 = lb1->lb_hash_config;
 397                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 398                     ip4_compute_flow_hash (ip1, flow_hash_config1);
 399             }
 400
 401             dpo0 = load_balance_get_bucket_i(lb0,
 402                                              (hash_c0 &
 403                                               (lb0->lb_n_buckets_minus_1)));
 404             dpo1 = load_balance_get_bucket_i(lb1,
 405                                              (hash_c1 &
 406                                               (lb1->lb_n_buckets_minus_1)));
 407
 408             next0 = dpo0->dpoi_next_node;
 409             next1 = dpo1->dpoi_next_node;
 410             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 411             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 412
 413             vlib_increment_combined_counter
 414                 (cm, cpu_index, lbi0, 1,
 415                  vlib_buffer_length_in_chain (vm, b0));
 416             vlib_increment_combined_counter
 417                 (cm, cpu_index, lbi1, 1,
 418                  vlib_buffer_length_in_chain (vm, b1));
 419
 420             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 421             {
 422                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 423                                                      b0, sizeof (*tr));
 424                 tr->fib_index = fib_index0;
 425                 tr->lbi = lbi0;
 426                 tr->addr.ip4 = *input_addr0;
 427             }
 428             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 429             {
 430                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 431                                                      b1, sizeof (*tr));
 432                 tr->fib_index = fib_index1;
 433                 tr->lbi = lbi1;
 434                 tr->addr.ip4 = *input_addr1;
 435             }
 436
 437             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 438                                              to_next, n_left_to_next,
 439                                              bi0, bi1, next0, next1);
 440         }
 441
 442         while (n_left_from > 0 && n_left_to_next > 0)
 443         {
 444             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 445             flow_hash_config_t flow_hash_config0;
 446             const ip4_address_t *input_addr;
 447             const load_balance_t *lb0;
 448             const lookup_dpo_t * lkd0;
 449             const ip4_header_t * ip0;
 450             const dpo_id_t *dpo0;
 451             vlib_buffer_t * b0;
 452
 453             bi0 = from[0];
 454             to_next[0] = bi0;
 455             from += 1;
 456             to_next += 1;
 457             n_left_from -= 1;
 458             n_left_to_next -= 1;
 459
 460             b0 = vlib_get_buffer (vm, bi0);
 461             ip0 = vlib_buffer_get_current (b0);
 462
 463             /* dst lookup was done by ip4 lookup */
 464             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 465             lkd0 = lookup_dpo_get(lkdi0);
 466
 467             /*
 468              * choose between a lookup using the fib index in the DPO
 469              * or getting the FIB index from the interface.
 470              */
 471             if (table_from_interface)
 472             {
 473                 fib_index0 =
 474                     ip4_fib_table_get_index_for_sw_if_index(
 475                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 476             }
 477             else
 478             {
 479                 fib_index0 = lkd0->lkd_fib_index;
 480             }
 481
 482             /*
 483              * choose between a source or destination address lookup in the table
 484              */
 485             if (input_src_addr)
 486             {
 487                 input_addr = &ip0->src_address;
 488             }
 489             else
 490             {
 491                 input_addr = &ip0->dst_address;
 492             }
 493
 494             /* do lookup */
 495             ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
 496             lb0 = load_balance_get(lbi0);
 497
 498             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 499
 500             /* Use flow hash to compute multipath adjacency. */
 501             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 502
 503             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 504             {
 505                 flow_hash_config0 = lb0->lb_hash_config;
 506                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 507                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 508             }
 509
 510             dpo0 = load_balance_get_bucket_i(lb0,
 511                                              (hash_c0 &
 512                                               (lb0->lb_n_buckets_minus_1)));
 513
 514             next0 = dpo0->dpoi_next_node;
 515             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 516
 517             vlib_increment_combined_counter
 518                 (cm, cpu_index, lbi0, 1,
 519                  vlib_buffer_length_in_chain (vm, b0));
 520
 521             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 522             {
 523                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 524                                                      b0, sizeof (*tr));
 525                 tr->fib_index = fib_index0;
 526                 tr->lbi = lbi0;
 527                 tr->addr.ip4 = *input_addr;
 528             }
 529
 530             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 531                                             n_left_to_next, bi0, next0);
 532         }
 533         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 534     }
 535     return from_frame->n_vectors;
 536 }
 537
 538 static u8 *
 539 format_lookup_trace (u8 * s, va_list * args)
 540 {
 541     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 542     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 543     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 544     uword indent = format_get_indent (s);
 545     s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
 546                 format_white_space, indent,
 547                 t->fib_index,
 548                 format_ip46_address, &t->addr, IP46_TYPE_ANY,
 549                 t->lbi);
 550     return s;
 551 }
 552
 553 always_inline uword
 554 lookup_ip4_dst (vlib_main_t * vm,
 555                 vlib_node_runtime_t * node,
 556                 vlib_frame_t * from_frame)
 557 {
 558     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
 559 }
 560
 561 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
 562     .function = lookup_ip4_dst,
 563     .name = "lookup-ip4-dst",
 564     .vector_size = sizeof (u32),
 565     .sibling_of = "ip4-lookup",
 566     .format_trace = format_lookup_trace,
 567 };
 568 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_node, lookup_ip4_dst)
 569
 570 always_inline uword
 571 lookup_ip4_dst_itf (vlib_main_t * vm,
 572                     vlib_node_runtime_t * node,
 573                     vlib_frame_t * from_frame)
 574 {
 575     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
 576 }
 577
 578 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
 579     .function = lookup_ip4_dst_itf,
 580     .name = "lookup-ip4-dst-itf",
 581     .vector_size = sizeof (u32),
 582     .sibling_of = "ip4-lookup",
 583     .format_trace = format_lookup_trace,
 584 };
 585 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_itf_node, lookup_ip4_dst_itf)
 586
 587 always_inline uword
 588 lookup_ip4_src (vlib_main_t * vm,
 589                 vlib_node_runtime_t * node,
 590                 vlib_frame_t * from_frame)
 591 {
 592     return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
 593 }
 594
 595 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
 596     .function = lookup_ip4_src,
 597     .name = "lookup-ip4-src",
 598     .vector_size = sizeof (u32),
 599     .format_trace = format_lookup_trace,
 600     .sibling_of = "ip4-lookup",
 601 };
 602 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_src_node, lookup_ip4_src)
 603
 604 always_inline uword
 605 lookup_dpo_ip6_inline (vlib_main_t * vm,
 606                        vlib_node_runtime_t * node,
 607                        vlib_frame_t * from_frame,
 608                        int input_src_addr,
 609                        int table_from_interface)
 610 {
 611     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 612     u32 n_left_from, next_index, * from, * to_next;
 613     u32 cpu_index = os_get_cpu_number();
 614
 615     from = vlib_frame_vector_args (from_frame);
 616     n_left_from = from_frame->n_vectors;
 617
 618     next_index = node->cached_next_index;
 619
 620     while (n_left_from > 0)
 621     {
 622         u32 n_left_to_next;
 623
 624         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 625
 626         while (n_left_from >= 4 && n_left_to_next > 2)
 627         {
 628             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 629             flow_hash_config_t flow_hash_config0;
 630             const ip6_address_t *input_addr0;
 631             const load_balance_t *lb0;
 632             const lookup_dpo_t * lkd0;
 633             const ip6_header_t * ip0;
 634             const dpo_id_t *dpo0;
 635             vlib_buffer_t * b0;
 636             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 637             flow_hash_config_t flow_hash_config1;
 638             const ip6_address_t *input_addr1;
 639             const load_balance_t *lb1;
 640             const lookup_dpo_t * lkd1;
 641             const ip6_header_t * ip1;
 642             const dpo_id_t *dpo1;
 643             vlib_buffer_t * b1;
 644
 645             /* Prefetch next iteration. */
 646             {
 647                 vlib_buffer_t * p2, * p3;
 648
 649                 p2 = vlib_get_buffer (vm, from[2]);
 650                 p3 = vlib_get_buffer (vm, from[3]);
 651
 652                 vlib_prefetch_buffer_header (p2, LOAD);
 653                 vlib_prefetch_buffer_header (p3, LOAD);
 654
 655                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 656                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 657             }
 658
 659             bi0 = from[0];
 660             to_next[0] = bi0;
 661             bi1 = from[1];
 662             to_next[1] = bi1;
 663             from += 2;
 664             to_next += 2;
 665             n_left_from -= 2;
 666             n_left_to_next -= 2;
 667
 668             b0 = vlib_get_buffer (vm, bi0);
 669             ip0 = vlib_buffer_get_current (b0);
 670             b1 = vlib_get_buffer (vm, bi1);
 671             ip1 = vlib_buffer_get_current (b1);
 672
 673             /* dst lookup was done by ip6 lookup */
 674             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 675             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 676             lkd0 = lookup_dpo_get(lkdi0);
 677             lkd1 = lookup_dpo_get(lkdi1);
 678
 679             /*
 680              * choose between a lookup using the fib index in the DPO
 681              * or getting the FIB index from the interface.
 682              */
 683             if (table_from_interface)
 684             {
 685                 fib_index0 =
 686                     ip6_fib_table_get_index_for_sw_if_index(
 687                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 688                 fib_index1 =
 689                     ip6_fib_table_get_index_for_sw_if_index(
 690                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 691             }
 692             else
 693             {
 694                 fib_index0 = lkd0->lkd_fib_index;
 695                 fib_index1 = lkd1->lkd_fib_index;
 696             }
 697
 698             /*
 699              * choose between a source or destination address lookup in the table
 700              */
 701             if (input_src_addr)
 702             {
 703                 input_addr0 = &ip0->src_address;
 704                 input_addr1 = &ip1->src_address;
 705             }
 706             else
 707             {
 708                 input_addr0 = &ip0->dst_address;
 709                 input_addr1 = &ip1->dst_address;
 710             }
 711
 712             /* do src lookup */
 713             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 714                                                fib_index0,
 715                                                input_addr0);
 716             lbi1 = ip6_fib_table_fwding_lookup(&ip6_main,
 717                                                fib_index1,
 718                                                input_addr1);
 719             lb0 = load_balance_get(lbi0);
 720             lb1 = load_balance_get(lbi1);
 721
 722             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 723             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 724
 725             /* Use flow hash to compute multipath adjacency. */
 726             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 727             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 728
 729             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 730             {
 731                 flow_hash_config0 = lb0->lb_hash_config;
 732                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 733                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 734             }
 735
 736             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 737             {
 738                 flow_hash_config1 = lb1->lb_hash_config;
 739                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 740                     ip6_compute_flow_hash (ip1, flow_hash_config1);
 741             }
 742
 743             dpo0 = load_balance_get_bucket_i(lb0,
 744                                              (hash_c0 &
 745                                               (lb0->lb_n_buckets_minus_1)));
 746             dpo1 = load_balance_get_bucket_i(lb1,
 747                                              (hash_c1 &
 748                                               (lb1->lb_n_buckets_minus_1)));
 749
 750             next0 = dpo0->dpoi_next_node;
 751             next1 = dpo1->dpoi_next_node;
 752             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 753             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 754
 755             vlib_increment_combined_counter
 756                 (cm, cpu_index, lbi0, 1,
 757                  vlib_buffer_length_in_chain (vm, b0));
 758             vlib_increment_combined_counter
 759                 (cm, cpu_index, lbi1, 1,
 760                  vlib_buffer_length_in_chain (vm, b1));
 761
 762             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 763             {
 764                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 765                                                      b0, sizeof (*tr));
 766                 tr->fib_index = fib_index0;
 767                 tr->lbi = lbi0;
 768                 tr->addr.ip6 = *input_addr0;
 769             }
 770             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 771             {
 772                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 773                                                      b1, sizeof (*tr));
 774                 tr->fib_index = fib_index1;
 775                 tr->lbi = lbi1;
 776                 tr->addr.ip6 = *input_addr1;
 777             }
 778             vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
 779                                             n_left_to_next, bi0, bi1,
 780                                             next0, next1);
 781         }
 782         while (n_left_from > 0 && n_left_to_next > 0)
 783         {
 784             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 785             flow_hash_config_t flow_hash_config0;
 786             const ip6_address_t *input_addr0;
 787             const load_balance_t *lb0;
 788             const lookup_dpo_t * lkd0;
 789             const ip6_header_t * ip0;
 790             const dpo_id_t *dpo0;
 791             vlib_buffer_t * b0;
 792
 793             bi0 = from[0];
 794             to_next[0] = bi0;
 795             from += 1;
 796             to_next += 1;
 797             n_left_from -= 1;
 798             n_left_to_next -= 1;
 799
 800             b0 = vlib_get_buffer (vm, bi0);
 801             ip0 = vlib_buffer_get_current (b0);
 802
 803             /* dst lookup was done by ip6 lookup */
 804             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 805             lkd0 = lookup_dpo_get(lkdi0);
 806
 807             /*
 808              * choose between a lookup using the fib index in the DPO
 809              * or getting the FIB index from the interface.
 810              */
 811             if (table_from_interface)
 812             {
 813                 fib_index0 =
 814                     ip6_fib_table_get_index_for_sw_if_index(
 815                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 816             }
 817             else
 818             {
 819                 fib_index0 = lkd0->lkd_fib_index;
 820             }
 821
 822             /*
 823              * choose between a source or destination address lookup in the table
 824              */
 825             if (input_src_addr)
 826             {
 827                 input_addr0 = &ip0->src_address;
 828             }
 829             else
 830             {
 831                 input_addr0 = &ip0->dst_address;
 832             }
 833
 834             /* do src lookup */
 835             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 836                                                fib_index0,
 837                                                input_addr0);
 838             lb0 = load_balance_get(lbi0);
 839
 840             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 841
 842             /* Use flow hash to compute multipath adjacency. */
 843             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 844
 845             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 846             {
 847                 flow_hash_config0 = lb0->lb_hash_config;
 848                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 849                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 850             }
 851
 852             dpo0 = load_balance_get_bucket_i(lb0,
 853                                              (hash_c0 &
 854                                               (lb0->lb_n_buckets_minus_1)));
 855
 856             next0 = dpo0->dpoi_next_node;
 857             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 858
 859             vlib_increment_combined_counter
 860                 (cm, cpu_index, lbi0, 1,
 861                  vlib_buffer_length_in_chain (vm, b0));
 862
 863             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 864             {
 865                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 866                                                      b0, sizeof (*tr));
 867                 tr->fib_index = fib_index0;
 868                 tr->lbi = lbi0;
 869                 tr->addr.ip6 = *input_addr0;
 870             }
 871             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 872                                             n_left_to_next, bi0, next0);
 873         }
 874         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 875     }
 876     return from_frame->n_vectors;
 877 }
 878
 879 always_inline uword
 880 lookup_ip6_dst (vlib_main_t * vm,
 881                 vlib_node_runtime_t * node,
 882                 vlib_frame_t * from_frame)
 883 {
 884     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 0));
 885 }
 886
 887 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
 888     .function = lookup_ip6_dst,
 889     .name = "lookup-ip6-dst",
 890     .vector_size = sizeof (u32),
 891     .format_trace = format_lookup_trace,
 892     .sibling_of = "ip6-lookup",
 893 };
 894 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_node, lookup_ip6_dst)
 895
 896 always_inline uword
 897 lookup_ip6_dst_itf (vlib_main_t * vm,
 898                     vlib_node_runtime_t * node,
 899                     vlib_frame_t * from_frame)
 900 {
 901     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 1));
 902 }
 903
 904 VLIB_REGISTER_NODE (lookup_ip6_dst_itf_node) = {
 905     .function = lookup_ip6_dst_itf,
 906     .name = "lookup-ip6-dst-itf",
 907     .vector_size = sizeof (u32),
 908     .format_trace = format_lookup_trace,
 909     .sibling_of = "ip6-lookup",
 910 };
 911 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_itf_node, lookup_ip6_dst_itf)
 912
 913 always_inline uword
 914 lookup_ip6_src (vlib_main_t * vm,
 915                 vlib_node_runtime_t * node,
 916                 vlib_frame_t * from_frame)
 917 {
 918     return (lookup_dpo_ip6_inline(vm, node, from_frame, 1, 0));
 919 }
 920
 921 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
 922     .function = lookup_ip6_src,
 923     .name = "lookup-ip6-src",
 924     .vector_size = sizeof (u32),
 925     .format_trace = format_lookup_trace,
 926     .sibling_of = "ip6-lookup",
 927 };
 928 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_src_node, lookup_ip6_src)
 929
 930 always_inline uword
 931 lookup_dpo_mpls_inline (vlib_main_t * vm,
 932                        vlib_node_runtime_t * node,
 933                        vlib_frame_t * from_frame,
 934                        int table_from_interface)
 935 {
 936     u32 n_left_from, next_index, * from, * to_next;
 937     u32 cpu_index = os_get_cpu_number();
 938     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 939
 940     from = vlib_frame_vector_args (from_frame);
 941     n_left_from = from_frame->n_vectors;
 942
 943     next_index = node->cached_next_index;
 944
 945     while (n_left_from > 0)
 946     {
 947         u32 n_left_to_next;
 948
 949         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 950
 951         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
 952         /*   } */
 953
 954         while (n_left_from > 0 && n_left_to_next > 0)
 955         {
 956             u32 bi0, lkdi0, lbi0, fib_index0,  next0;
 957             const mpls_unicast_header_t * hdr0;
 958             const load_balance_t *lb0;
 959             const lookup_dpo_t * lkd0;
 960             const dpo_id_t *dpo0;
 961             vlib_buffer_t * b0;
 962
 963             bi0 = from[0];
 964             to_next[0] = bi0;
 965             from += 1;
 966             to_next += 1;
 967             n_left_from -= 1;
 968             n_left_to_next -= 1;
 969
 970             b0 = vlib_get_buffer (vm, bi0);
 971             hdr0 = vlib_buffer_get_current (b0);
 972
 973             /* dst lookup was done by mpls lookup */
 974             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 975             lkd0 = lookup_dpo_get(lkdi0);
 976
 977             /*
 978              * choose between a lookup using the fib index in the DPO
 979              * or getting the FIB index from the interface.
 980              */
 981             if (table_from_interface)
 982             {
 983                 fib_index0 =
 984                     mpls_fib_table_get_index_for_sw_if_index(
 985                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 986             }
 987             else
 988             {
 989                 fib_index0 = lkd0->lkd_fib_index;
 990             }
 991
 992             /* do lookup */
 993             lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
 994             lb0  = load_balance_get(lbi0);
 995             dpo0 = load_balance_get_bucket_i(lb0, 0);
 996
 997             next0 = dpo0->dpoi_next_node;
 998             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 999
1000             vlib_increment_combined_counter
1001                 (cm, cpu_index, lbi0, 1,
1002                  vlib_buffer_length_in_chain (vm, b0));
1003
1004             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1005             {
1006                 lookup_trace_t *tr = vlib_add_trace (vm, node,
1007                                                      b0, sizeof (*tr));
1008                 tr->fib_index = fib_index0;
1009                 tr->lbi = lbi0;
1010                 tr->hdr = *hdr0;
1011             }
1012
1013            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1014                                             n_left_to_next, bi0, next0);
1015         }
1016         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1017     }
1018     return from_frame->n_vectors;
1019 }
1020
1021 static u8 *
1022 format_lookup_mpls_trace (u8 * s, va_list * args)
1023 {
1024     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1025     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1026     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
1027     uword indent = format_get_indent (s);
1028     mpls_unicast_header_t hdr;
1029
1030     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
1031
1032     s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
1033                 format_white_space, indent,
1034                 t->fib_index,
1035                 format_mpls_header, hdr,
1036                 t->lbi);
1037     return s;
1038 }
1039
1040 always_inline uword
1041 lookup_mpls_dst (vlib_main_t * vm,
1042                 vlib_node_runtime_t * node,
1043                 vlib_frame_t * from_frame)
1044 {
1045     return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
1046 }
1047
1048 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
1049     .function = lookup_mpls_dst,
1050     .name = "lookup-mpls-dst",
1051     .vector_size = sizeof (u32),
1052     .sibling_of = "mpls-lookup",
1053     .format_trace = format_lookup_mpls_trace,
1054     .n_next_nodes = 0,
1055 };
1056 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_node, lookup_mpls_dst)
1057
1058 always_inline uword
1059 lookup_mpls_dst_itf (vlib_main_t * vm,
1060                     vlib_node_runtime_t * node,
1061                     vlib_frame_t * from_frame)
1062 {
1063     return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
1064 }
1065
1066 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
1067     .function = lookup_mpls_dst_itf,
1068     .name = "lookup-mpls-dst-itf",
1069     .vector_size = sizeof (u32),
1070     .sibling_of = "mpls-lookup",
1071     .format_trace = format_lookup_mpls_trace,
1072     .n_next_nodes = 0,
1073 };
1074 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf)
1075
1076 static void
1077 lookup_dpo_mem_show (void)
1078 {
1079     fib_show_memory_usage("Lookup",
1080                           pool_elts(lookup_dpo_pool),
1081                           pool_len(lookup_dpo_pool),
1082                           sizeof(lookup_dpo_t));
1083 }
1084
1085 const static dpo_vft_t lkd_vft = {
1086     .dv_lock = lookup_dpo_lock,
1087     .dv_unlock = lookup_dpo_unlock,
1088     .dv_format = format_lookup_dpo,
1089 };
1090 const static dpo_vft_t lkd_vft_w_mem_show = {
1091     .dv_lock = lookup_dpo_lock,
1092     .dv_unlock = lookup_dpo_unlock,
1093     .dv_format = format_lookup_dpo,
1094     .dv_mem_show = lookup_dpo_mem_show,
1095 };
1096
1097 const static char* const lookup_src_ip4_nodes[] =
1098 {
1099     "lookup-ip4-src",
1100     NULL,
1101 };
1102 const static char* const lookup_src_ip6_nodes[] =
1103 {
1104     "lookup-ip6-src",
1105     NULL,
1106 };
1107 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
1108 {
1109     [DPO_PROTO_IP4]  = lookup_src_ip4_nodes,
1110     [DPO_PROTO_IP6]  = lookup_src_ip6_nodes,
1111     [DPO_PROTO_MPLS] = NULL,
1112 };
1113
1114 const static char* const lookup_dst_ip4_nodes[] =
1115 {
1116     "lookup-ip4-dst",
1117     NULL,
1118 };
1119 const static char* const lookup_dst_ip6_nodes[] =
1120 {
1121     "lookup-ip6-dst",
1122     NULL,
1123 };
1124 const static char* const lookup_dst_mpls_nodes[] =
1125 {
1126     "lookup-mpls-dst",
1127     NULL,
1128 };
1129 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
1130 {
1131     [DPO_PROTO_IP4]  = lookup_dst_ip4_nodes,
1132     [DPO_PROTO_IP6]  = lookup_dst_ip6_nodes,
1133     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
1134 };
1135
1136 const static char* const lookup_dst_from_interface_ip4_nodes[] =
1137 {
1138     "lookup-ip4-dst-itf",
1139     NULL,
1140 };
1141 const static char* const lookup_dst_from_interface_ip6_nodes[] =
1142 {
1143     "lookup-ip6-dst-itf",
1144     NULL,
1145 };
1146 const static char* const lookup_dst_from_interface_mpls_nodes[] =
1147 {
1148     "lookup-mpls-dst-itf",
1149     NULL,
1150 };
1151 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
1152 {
1153     [DPO_PROTO_IP4]  = lookup_dst_from_interface_ip4_nodes,
1154     [DPO_PROTO_IP6]  = lookup_dst_from_interface_ip6_nodes,
1155     [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
1156 };
1157
1158
1159 void
1160 lookup_dpo_module_init (void)
1161 {
1162     dpo_register(DPO_LOOKUP, &lkd_vft_w_mem_show, NULL);
1163
1164     /*
1165      * There are various sorts of lookup; src or dst addr v4 /v6 etc.
1166      * there isn't an object type for each (there is only the lookup_dpo_t),
1167      * but, for performance reasons, there is a data plane function, and hence
1168      * VLIB node for each. VLIB graph node construction is based on DPO types
1169      * so we create sub-types.
1170      */
1171     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
1172         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
1173     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
1174         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
1175     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
1176         dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes);
1177 }