src/vnet/dpo/lookup_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/lookup_dpo.h>
  18 #include <vnet/dpo/load_balance_map.h>
  19 #include <vnet/mpls/mpls_lookup.h>
  20 #include <vnet/fib/fib_table.h>
  21 #include <vnet/fib/ip4_fib.h>
  22 #include <vnet/fib/ip6_fib.h>
  23 #include <vnet/fib/mpls_fib.h>
  24 #include <vnet/mfib/mfib_table.h>
  25 #include <vnet/mfib/ip4_mfib.h>
  26 #include <vnet/mfib/ip6_mfib.h>
  27
  28 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
  29 static const char *const lookup_cast_names[] = LOOKUP_CASTS;
  30
  31 /**
  32  * @brief Enumeration of the lookup subtypes
  33  */
  34 typedef enum lookup_sub_type_t_
  35 {
  36     LOOKUP_SUB_TYPE_SRC,
  37     LOOKUP_SUB_TYPE_DST,
  38     LOOKUP_SUB_TYPE_DST_MCAST,
  39     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
  40 } lookup_sub_type_t;
  41 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
  42
  43 #define FOR_EACH_LOOKUP_SUB_TYPE(_st)                                   \
  44     for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
  45
  46 /**
  47  * @brief pool of all MPLS Label DPOs
  48  */
  49 lookup_dpo_t *lookup_dpo_pool;
  50
  51 /**
  52  * @brief An array of registered DPO type values for the sub-types
  53  */
  54 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
  55
  56 static lookup_dpo_t *
  57 lookup_dpo_alloc (void)
  58 {
  59     lookup_dpo_t *lkd;
  60
  61     pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
  62
  63     return (lkd);
  64 }
  65
  66 static index_t
  67 lookup_dpo_get_index (lookup_dpo_t *lkd)
  68 {
  69     return (lkd - lookup_dpo_pool);
  70 }
  71
  72 static void
  73 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
  74                           dpo_proto_t proto,
  75                           lookup_cast_t cast,
  76                           lookup_input_t input,
  77                           lookup_table_t table_config,
  78                           dpo_id_t *dpo)
  79 {
  80     lookup_dpo_t *lkd;
  81     dpo_type_t type;
  82
  83     lkd = lookup_dpo_alloc();
  84     lkd->lkd_fib_index = fib_index;
  85     lkd->lkd_proto = proto;
  86     lkd->lkd_input = input;
  87     lkd->lkd_table = table_config;
  88     lkd->lkd_cast  = cast;
  89
  90     /*
  91      * use the input type to select the lookup sub-type
  92      */
  93     type = 0;
  94
  95     switch (input)
  96     {
  97     case LOOKUP_INPUT_SRC_ADDR:
  98         type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
  99         break;
 100     case LOOKUP_INPUT_DST_ADDR:
 101         switch (table_config)
 102         {
 103         case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
 104             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
 105             break;
 106         case LOOKUP_TABLE_FROM_CONFIG:
 107             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
 108             break;
 109         }
 110         if (LOOKUP_MULTICAST == cast)
 111         {
 112             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST];
 113         }
 114     }
 115
 116     if (0 == type)
 117     {
 118         dpo_reset(dpo);
 119     }
 120     else
 121     {
 122         dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
 123     }
 124 }
 125
 126 void
 127 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
 128                                     dpo_proto_t proto,
 129                                     lookup_cast_t cast,
 130                                     lookup_input_t input,
 131                                     lookup_table_t table_config,
 132                                     dpo_id_t *dpo)
 133 {
 134     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 135     {
 136         if (LOOKUP_UNICAST == cast)
 137         {
 138             fib_table_lock(fib_index,
 139                            dpo_proto_to_fib(proto),
 140                            FIB_SOURCE_RR);
 141         }
 142         else
 143         {
 144             mfib_table_lock(fib_index,
 145                             dpo_proto_to_fib(proto),
 146                             MFIB_SOURCE_RR);
 147         }
 148     }
 149     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 150 }
 151
 152 void
 153 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 154                                    dpo_proto_t proto,
 155                                    lookup_cast_t cast,
 156                                    lookup_input_t input,
 157                                    lookup_table_t table_config,
 158                                    dpo_id_t *dpo)
 159 {
 160     fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
 161
 162     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 163     {
 164         if (LOOKUP_UNICAST == cast)
 165         {
 166             fib_index =
 167                 fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 168                                                   table_id,
 169                                                   FIB_SOURCE_RR);
 170         }
 171         else
 172         {
 173             fib_index =
 174                 mfib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 175                                                    table_id,
 176                                                    MFIB_SOURCE_RR);
 177         }
 178     }
 179
 180     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
 181     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 182 }
 183
 184 u8*
 185 format_lookup_dpo (u8 *s, va_list *args)
 186 {
 187     index_t index = va_arg (*args, index_t);
 188     lookup_dpo_t *lkd;
 189
 190     lkd = lookup_dpo_get(index);
 191
 192     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
 193     {
 194         s = format(s, "%s,%s lookup in interface's %U table",
 195                    lookup_input_names[lkd->lkd_input],
 196                    lookup_cast_names[lkd->lkd_cast],
 197                    format_dpo_proto, lkd->lkd_proto);
 198     }
 199     else
 200     {
 201         if (LOOKUP_UNICAST == lkd->lkd_cast)
 202         {
 203             s = format(s, "%s,%s lookup in %U",
 204                        lookup_input_names[lkd->lkd_input],
 205                        lookup_cast_names[lkd->lkd_cast],
 206                        format_fib_table_name, lkd->lkd_fib_index,
 207                        dpo_proto_to_fib(lkd->lkd_proto));
 208         }
 209         else
 210         {
 211             s = format(s, "%s,%s lookup in %U",
 212                        lookup_input_names[lkd->lkd_input],
 213                        lookup_cast_names[lkd->lkd_cast],
 214                        format_mfib_table_name, lkd->lkd_fib_index,
 215                        dpo_proto_to_fib(lkd->lkd_proto));
 216         }
 217     }
 218     return (s);
 219 }
 220
 221 static void
 222 lookup_dpo_lock (dpo_id_t *dpo)
 223 {
 224     lookup_dpo_t *lkd;
 225
 226     lkd = lookup_dpo_get(dpo->dpoi_index);
 227
 228     lkd->lkd_locks++;
 229 }
 230
 231 static void
 232 lookup_dpo_unlock (dpo_id_t *dpo)
 233 {
 234     lookup_dpo_t *lkd;
 235
 236     lkd = lookup_dpo_get(dpo->dpoi_index);
 237
 238     lkd->lkd_locks--;
 239
 240     if (0 == lkd->lkd_locks)
 241     {
 242         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
 243         {
 244             if (LOOKUP_UNICAST == lkd->lkd_cast)
 245             {
 246                 fib_table_unlock(lkd->lkd_fib_index,
 247                                  dpo_proto_to_fib(lkd->lkd_proto),
 248                                  FIB_SOURCE_RR);
 249             }
 250             else
 251             {
 252                 mfib_table_unlock(lkd->lkd_fib_index,
 253                                   dpo_proto_to_fib(lkd->lkd_proto),
 254                                   MFIB_SOURCE_RR);
 255             }
 256         }
 257         pool_put(lookup_dpo_pool, lkd);
 258     }
 259 }
 260
 261 always_inline void
 262 ip4_src_fib_lookup_one (u32 src_fib_index0,
 263                         const ip4_address_t * addr0,
 264                         u32 * src_adj_index0)
 265 {
 266     ip4_fib_mtrie_leaf_t leaf0;
 267     ip4_fib_mtrie_t * mtrie0;
 268
 269     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 270
 271     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 272     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 273     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 274
 275     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 276 }
 277
 278 always_inline void
 279 ip4_src_fib_lookup_two (u32 src_fib_index0,
 280                         u32 src_fib_index1,
 281                         const ip4_address_t * addr0,
 282                         const ip4_address_t * addr1,
 283                         u32 * src_adj_index0,
 284                         u32 * src_adj_index1)
 285 {
 286     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 287     ip4_fib_mtrie_t * mtrie0, * mtrie1;
 288
 289     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 290     mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
 291
 292     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 293     leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, addr1);
 294
 295     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 296     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
 297
 298     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 299     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
 300
 301     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 302     src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 303 }
 304
 305 /**
 306  * @brief Lookup trace  data
 307  */
 308 typedef struct lookup_trace_t_
 309 {
 310     union {
 311         ip46_address_t addr;
 312         mpls_unicast_header_t hdr;
 313     };
 314     fib_node_index_t fib_index;
 315     index_t lbi;
 316 } lookup_trace_t;
 317
 318
 319 always_inline uword
 320 lookup_dpo_ip4_inline (vlib_main_t * vm,
 321                        vlib_node_runtime_t * node,
 322                        vlib_frame_t * from_frame,
 323                        int input_src_addr,
 324                        int table_from_interface)
 325 {
 326     u32 n_left_from, next_index, * from, * to_next;
 327     u32 thread_index = vlib_get_thread_index();
 328     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 329
 330     from = vlib_frame_vector_args (from_frame);
 331     n_left_from = from_frame->n_vectors;
 332
 333     next_index = node->cached_next_index;
 334
 335     while (n_left_from > 0)
 336     {
 337         u32 n_left_to_next;
 338
 339         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 340
 341         while (n_left_from >= 4 && n_left_to_next > 2)
 342         {
 343             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 344             flow_hash_config_t flow_hash_config0;
 345             const ip4_address_t *input_addr0;
 346             const load_balance_t *lb0;
 347             const lookup_dpo_t * lkd0;
 348             const ip4_header_t * ip0;
 349             const dpo_id_t *dpo0;
 350             vlib_buffer_t * b0;
 351             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 352             flow_hash_config_t flow_hash_config1;
 353             const ip4_address_t *input_addr1;
 354             const load_balance_t *lb1;
 355             const lookup_dpo_t * lkd1;
 356             const ip4_header_t * ip1;
 357             const dpo_id_t *dpo1;
 358             vlib_buffer_t * b1;
 359
 360             /* Prefetch next iteration. */
 361             {
 362                 vlib_buffer_t * p2, * p3;
 363
 364                 p2 = vlib_get_buffer (vm, from[2]);
 365                 p3 = vlib_get_buffer (vm, from[3]);
 366
 367                 vlib_prefetch_buffer_header (p2, LOAD);
 368                 vlib_prefetch_buffer_header (p3, LOAD);
 369
 370                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 371                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 372             }
 373
 374             bi0 = from[0];
 375             to_next[0] = bi0;
 376             bi1 = from[1];
 377             to_next[1] = bi1;
 378             from += 2;
 379             to_next += 2;
 380             n_left_from -= 2;
 381             n_left_to_next -= 2;
 382
 383             b0 = vlib_get_buffer (vm, bi0);
 384             ip0 = vlib_buffer_get_current (b0);
 385             b1 = vlib_get_buffer (vm, bi1);
 386             ip1 = vlib_buffer_get_current (b1);
 387
 388             /* dst lookup was done by ip4 lookup */
 389             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 390             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 391             lkd0 = lookup_dpo_get(lkdi0);
 392             lkd1 = lookup_dpo_get(lkdi1);
 393
 394             /*
 395              * choose between a lookup using the fib index in the DPO
 396              * or getting the FIB index from the interface.
 397              */
 398             if (table_from_interface)
 399             {
 400                 fib_index0 =
 401                     ip4_fib_table_get_index_for_sw_if_index(
 402                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 403                 fib_index1 =
 404                     ip4_fib_table_get_index_for_sw_if_index(
 405                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 406             }
 407             else
 408             {
 409                 fib_index0 = lkd0->lkd_fib_index;
 410                 fib_index1 = lkd1->lkd_fib_index;
 411             }
 412
 413             /*
 414              * choose between a source or destination address lookup in the table
 415              */
 416             if (input_src_addr)
 417             {
 418                 input_addr0 = &ip0->src_address;
 419                 input_addr1 = &ip1->src_address;
 420             }
 421             else
 422             {
 423                 input_addr0 = &ip0->dst_address;
 424                 input_addr1 = &ip1->dst_address;
 425             }
 426
 427             /* do lookup */
 428             ip4_src_fib_lookup_two (fib_index0, fib_index1,
 429                                     input_addr0, input_addr1,
 430                                     &lbi0, &lbi1);
 431             lb0 = load_balance_get(lbi0);
 432             lb1 = load_balance_get(lbi1);
 433
 434             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 435             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 436
 437             /* Use flow hash to compute multipath adjacency. */
 438             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 439             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 440
 441             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 442             {
 443                 flow_hash_config0 = lb0->lb_hash_config;
 444                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 445                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 446             }
 447
 448             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 449             {
 450                 flow_hash_config1 = lb1->lb_hash_config;
 451                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 452                     ip4_compute_flow_hash (ip1, flow_hash_config1);
 453             }
 454
 455             dpo0 = load_balance_get_bucket_i(lb0,
 456                                              (hash_c0 &
 457                                               (lb0->lb_n_buckets_minus_1)));
 458             dpo1 = load_balance_get_bucket_i(lb1,
 459                                              (hash_c1 &
 460                                               (lb1->lb_n_buckets_minus_1)));
 461
 462             next0 = dpo0->dpoi_next_node;
 463             next1 = dpo1->dpoi_next_node;
 464             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 465             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 466
 467             vlib_increment_combined_counter
 468                 (cm, thread_index, lbi0, 1,
 469                  vlib_buffer_length_in_chain (vm, b0));
 470             vlib_increment_combined_counter
 471                 (cm, thread_index, lbi1, 1,
 472                  vlib_buffer_length_in_chain (vm, b1));
 473
 474             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 475             {
 476                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 477                                                      b0, sizeof (*tr));
 478                 tr->fib_index = fib_index0;
 479                 tr->lbi = lbi0;
 480                 tr->addr.ip4 = *input_addr0;
 481             }
 482             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 483             {
 484                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 485                                                      b1, sizeof (*tr));
 486                 tr->fib_index = fib_index1;
 487                 tr->lbi = lbi1;
 488                 tr->addr.ip4 = *input_addr1;
 489             }
 490
 491             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 492                                              to_next, n_left_to_next,
 493                                              bi0, bi1, next0, next1);
 494         }
 495
 496         while (n_left_from > 0 && n_left_to_next > 0)
 497         {
 498             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 499             flow_hash_config_t flow_hash_config0;
 500             const ip4_address_t *input_addr;
 501             const load_balance_t *lb0;
 502             const lookup_dpo_t * lkd0;
 503             const ip4_header_t * ip0;
 504             const dpo_id_t *dpo0;
 505             vlib_buffer_t * b0;
 506
 507             bi0 = from[0];
 508             to_next[0] = bi0;
 509             from += 1;
 510             to_next += 1;
 511             n_left_from -= 1;
 512             n_left_to_next -= 1;
 513
 514             b0 = vlib_get_buffer (vm, bi0);
 515             ip0 = vlib_buffer_get_current (b0);
 516
 517             /* dst lookup was done by ip4 lookup */
 518             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 519             lkd0 = lookup_dpo_get(lkdi0);
 520
 521             /*
 522              * choose between a lookup using the fib index in the DPO
 523              * or getting the FIB index from the interface.
 524              */
 525             if (table_from_interface)
 526             {
 527                 fib_index0 =
 528                     ip4_fib_table_get_index_for_sw_if_index(
 529                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 530             }
 531             else
 532             {
 533                 fib_index0 = lkd0->lkd_fib_index;
 534             }
 535
 536             /*
 537              * choose between a source or destination address lookup in the table
 538              */
 539             if (input_src_addr)
 540             {
 541                 input_addr = &ip0->src_address;
 542             }
 543             else
 544             {
 545                 input_addr = &ip0->dst_address;
 546             }
 547
 548             /* do lookup */
 549             ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
 550             lb0 = load_balance_get(lbi0);
 551
 552             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 553
 554             /* Use flow hash to compute multipath adjacency. */
 555             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 556
 557             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 558             {
 559                 flow_hash_config0 = lb0->lb_hash_config;
 560                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 561                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 562             }
 563
 564             dpo0 = load_balance_get_bucket_i(lb0,
 565                                              (hash_c0 &
 566                                               (lb0->lb_n_buckets_minus_1)));
 567
 568             next0 = dpo0->dpoi_next_node;
 569             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 570
 571             vlib_increment_combined_counter
 572                 (cm, thread_index, lbi0, 1,
 573                  vlib_buffer_length_in_chain (vm, b0));
 574
 575             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 576             {
 577                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 578                                                      b0, sizeof (*tr));
 579                 tr->fib_index = fib_index0;
 580                 tr->lbi = lbi0;
 581                 tr->addr.ip4 = *input_addr;
 582             }
 583
 584             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 585                                             n_left_to_next, bi0, next0);
 586         }
 587         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 588     }
 589     return from_frame->n_vectors;
 590 }
 591
 592 static u8 *
 593 format_lookup_trace (u8 * s, va_list * args)
 594 {
 595     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 596     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 597     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 598     uword indent = format_get_indent (s);
 599     s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
 600                 format_white_space, indent,
 601                 t->fib_index,
 602                 format_ip46_address, &t->addr, IP46_TYPE_ANY,
 603                 t->lbi);
 604     return s;
 605 }
 606
 607 always_inline uword
 608 lookup_ip4_dst (vlib_main_t * vm,
 609                 vlib_node_runtime_t * node,
 610                 vlib_frame_t * from_frame)
 611 {
 612     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
 613 }
 614
 615 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
 616     .function = lookup_ip4_dst,
 617     .name = "lookup-ip4-dst",
 618     .vector_size = sizeof (u32),
 619     .sibling_of = "ip4-lookup",
 620     .format_trace = format_lookup_trace,
 621 };
 622 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_node, lookup_ip4_dst)
 623
 624 always_inline uword
 625 lookup_ip4_dst_itf (vlib_main_t * vm,
 626                     vlib_node_runtime_t * node,
 627                     vlib_frame_t * from_frame)
 628 {
 629     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
 630 }
 631
 632 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
 633     .function = lookup_ip4_dst_itf,
 634     .name = "lookup-ip4-dst-itf",
 635     .vector_size = sizeof (u32),
 636     .sibling_of = "ip4-lookup",
 637     .format_trace = format_lookup_trace,
 638 };
 639 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_itf_node, lookup_ip4_dst_itf)
 640
 641 always_inline uword
 642 lookup_ip4_src (vlib_main_t * vm,
 643                 vlib_node_runtime_t * node,
 644                 vlib_frame_t * from_frame)
 645 {
 646     return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
 647 }
 648
 649 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
 650     .function = lookup_ip4_src,
 651     .name = "lookup-ip4-src",
 652     .vector_size = sizeof (u32),
 653     .format_trace = format_lookup_trace,
 654     .sibling_of = "ip4-lookup",
 655 };
 656 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_src_node, lookup_ip4_src)
 657
 658 always_inline uword
 659 lookup_dpo_ip6_inline (vlib_main_t * vm,
 660                        vlib_node_runtime_t * node,
 661                        vlib_frame_t * from_frame,
 662                        int input_src_addr,
 663                        int table_from_interface)
 664 {
 665     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 666     u32 n_left_from, next_index, * from, * to_next;
 667     u32 thread_index = vlib_get_thread_index();
 668
 669     from = vlib_frame_vector_args (from_frame);
 670     n_left_from = from_frame->n_vectors;
 671
 672     next_index = node->cached_next_index;
 673
 674     while (n_left_from > 0)
 675     {
 676         u32 n_left_to_next;
 677
 678         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 679
 680         while (n_left_from >= 4 && n_left_to_next > 2)
 681         {
 682             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 683             flow_hash_config_t flow_hash_config0;
 684             const ip6_address_t *input_addr0;
 685             const load_balance_t *lb0;
 686             const lookup_dpo_t * lkd0;
 687             const ip6_header_t * ip0;
 688             const dpo_id_t *dpo0;
 689             vlib_buffer_t * b0;
 690             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 691             flow_hash_config_t flow_hash_config1;
 692             const ip6_address_t *input_addr1;
 693             const load_balance_t *lb1;
 694             const lookup_dpo_t * lkd1;
 695             const ip6_header_t * ip1;
 696             const dpo_id_t *dpo1;
 697             vlib_buffer_t * b1;
 698
 699             /* Prefetch next iteration. */
 700             {
 701                 vlib_buffer_t * p2, * p3;
 702
 703                 p2 = vlib_get_buffer (vm, from[2]);
 704                 p3 = vlib_get_buffer (vm, from[3]);
 705
 706                 vlib_prefetch_buffer_header (p2, LOAD);
 707                 vlib_prefetch_buffer_header (p3, LOAD);
 708
 709                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 710                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 711             }
 712
 713             bi0 = from[0];
 714             to_next[0] = bi0;
 715             bi1 = from[1];
 716             to_next[1] = bi1;
 717             from += 2;
 718             to_next += 2;
 719             n_left_from -= 2;
 720             n_left_to_next -= 2;
 721
 722             b0 = vlib_get_buffer (vm, bi0);
 723             ip0 = vlib_buffer_get_current (b0);
 724             b1 = vlib_get_buffer (vm, bi1);
 725             ip1 = vlib_buffer_get_current (b1);
 726
 727             /* dst lookup was done by ip6 lookup */
 728             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 729             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 730             lkd0 = lookup_dpo_get(lkdi0);
 731             lkd1 = lookup_dpo_get(lkdi1);
 732
 733             /*
 734              * choose between a lookup using the fib index in the DPO
 735              * or getting the FIB index from the interface.
 736              */
 737             if (table_from_interface)
 738             {
 739                 fib_index0 =
 740                     ip6_fib_table_get_index_for_sw_if_index(
 741                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 742                 fib_index1 =
 743                     ip6_fib_table_get_index_for_sw_if_index(
 744                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 745             }
 746             else
 747             {
 748                 fib_index0 = lkd0->lkd_fib_index;
 749                 fib_index1 = lkd1->lkd_fib_index;
 750             }
 751
 752             /*
 753              * choose between a source or destination address lookup in the table
 754              */
 755             if (input_src_addr)
 756             {
 757                 input_addr0 = &ip0->src_address;
 758                 input_addr1 = &ip1->src_address;
 759             }
 760             else
 761             {
 762                 input_addr0 = &ip0->dst_address;
 763                 input_addr1 = &ip1->dst_address;
 764             }
 765
 766             /* do src lookup */
 767             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 768                                                fib_index0,
 769                                                input_addr0);
 770             lbi1 = ip6_fib_table_fwding_lookup(&ip6_main,
 771                                                fib_index1,
 772                                                input_addr1);
 773             lb0 = load_balance_get(lbi0);
 774             lb1 = load_balance_get(lbi1);
 775
 776             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 777             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 778
 779             /* Use flow hash to compute multipath adjacency. */
 780             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 781             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 782
 783             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 784             {
 785                 flow_hash_config0 = lb0->lb_hash_config;
 786                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 787                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 788             }
 789
 790             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 791             {
 792                 flow_hash_config1 = lb1->lb_hash_config;
 793                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 794                     ip6_compute_flow_hash (ip1, flow_hash_config1);
 795             }
 796
 797             dpo0 = load_balance_get_bucket_i(lb0,
 798                                              (hash_c0 &
 799                                               (lb0->lb_n_buckets_minus_1)));
 800             dpo1 = load_balance_get_bucket_i(lb1,
 801                                              (hash_c1 &
 802                                               (lb1->lb_n_buckets_minus_1)));
 803
 804             next0 = dpo0->dpoi_next_node;
 805             next1 = dpo1->dpoi_next_node;
 806             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 807             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 808
 809             vlib_increment_combined_counter
 810                 (cm, thread_index, lbi0, 1,
 811                  vlib_buffer_length_in_chain (vm, b0));
 812             vlib_increment_combined_counter
 813                 (cm, thread_index, lbi1, 1,
 814                  vlib_buffer_length_in_chain (vm, b1));
 815
 816             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 817             {
 818                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 819                                                      b0, sizeof (*tr));
 820                 tr->fib_index = fib_index0;
 821                 tr->lbi = lbi0;
 822                 tr->addr.ip6 = *input_addr0;
 823             }
 824             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 825             {
 826                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 827                                                      b1, sizeof (*tr));
 828                 tr->fib_index = fib_index1;
 829                 tr->lbi = lbi1;
 830                 tr->addr.ip6 = *input_addr1;
 831             }
 832             vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
 833                                             n_left_to_next, bi0, bi1,
 834                                             next0, next1);
 835         }
 836         while (n_left_from > 0 && n_left_to_next > 0)
 837         {
 838             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 839             flow_hash_config_t flow_hash_config0;
 840             const ip6_address_t *input_addr0;
 841             const load_balance_t *lb0;
 842             const lookup_dpo_t * lkd0;
 843             const ip6_header_t * ip0;
 844             const dpo_id_t *dpo0;
 845             vlib_buffer_t * b0;
 846
 847             bi0 = from[0];
 848             to_next[0] = bi0;
 849             from += 1;
 850             to_next += 1;
 851             n_left_from -= 1;
 852             n_left_to_next -= 1;
 853
 854             b0 = vlib_get_buffer (vm, bi0);
 855             ip0 = vlib_buffer_get_current (b0);
 856
 857             /* dst lookup was done by ip6 lookup */
 858             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 859             lkd0 = lookup_dpo_get(lkdi0);
 860
 861             /*
 862              * choose between a lookup using the fib index in the DPO
 863              * or getting the FIB index from the interface.
 864              */
 865             if (table_from_interface)
 866             {
 867                 fib_index0 =
 868                     ip6_fib_table_get_index_for_sw_if_index(
 869                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 870             }
 871             else
 872             {
 873                 fib_index0 = lkd0->lkd_fib_index;
 874             }
 875
 876             /*
 877              * choose between a source or destination address lookup in the table
 878              */
 879             if (input_src_addr)
 880             {
 881                 input_addr0 = &ip0->src_address;
 882             }
 883             else
 884             {
 885                 input_addr0 = &ip0->dst_address;
 886             }
 887
 888             /* do src lookup */
 889             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 890                                                fib_index0,
 891                                                input_addr0);
 892             lb0 = load_balance_get(lbi0);
 893
 894             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 895
 896             /* Use flow hash to compute multipath adjacency. */
 897             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 898
 899             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 900             {
 901                 flow_hash_config0 = lb0->lb_hash_config;
 902                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 903                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 904             }
 905
 906             dpo0 = load_balance_get_bucket_i(lb0,
 907                                              (hash_c0 &
 908                                               (lb0->lb_n_buckets_minus_1)));
 909
 910             next0 = dpo0->dpoi_next_node;
 911             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 912
 913             vlib_increment_combined_counter
 914                 (cm, thread_index, lbi0, 1,
 915                  vlib_buffer_length_in_chain (vm, b0));
 916
 917             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 918             {
 919                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 920                                                      b0, sizeof (*tr));
 921                 tr->fib_index = fib_index0;
 922                 tr->lbi = lbi0;
 923                 tr->addr.ip6 = *input_addr0;
 924             }
 925             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 926                                             n_left_to_next, bi0, next0);
 927         }
 928         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 929     }
 930     return from_frame->n_vectors;
 931 }
 932
 933 always_inline uword
 934 lookup_ip6_dst (vlib_main_t * vm,
 935                 vlib_node_runtime_t * node,
 936                 vlib_frame_t * from_frame)
 937 {
 938     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 0));
 939 }
 940
 941 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
 942     .function = lookup_ip6_dst,
 943     .name = "lookup-ip6-dst",
 944     .vector_size = sizeof (u32),
 945     .format_trace = format_lookup_trace,
 946     .sibling_of = "ip6-lookup",
 947 };
 948 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_node, lookup_ip6_dst)
 949
 950 always_inline uword
 951 lookup_ip6_dst_itf (vlib_main_t * vm,
 952                     vlib_node_runtime_t * node,
 953                     vlib_frame_t * from_frame)
 954 {
 955     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 1));
 956 }
 957
 958 VLIB_REGISTER_NODE (lookup_ip6_dst_itf_node) = {
 959     .function = lookup_ip6_dst_itf,
 960     .name = "lookup-ip6-dst-itf",
 961     .vector_size = sizeof (u32),
 962     .format_trace = format_lookup_trace,
 963     .sibling_of = "ip6-lookup",
 964 };
 965 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_itf_node, lookup_ip6_dst_itf)
 966
 967 always_inline uword
 968 lookup_ip6_src (vlib_main_t * vm,
 969                 vlib_node_runtime_t * node,
 970                 vlib_frame_t * from_frame)
 971 {
 972     return (lookup_dpo_ip6_inline(vm, node, from_frame, 1, 0));
 973 }
 974
 975 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
 976     .function = lookup_ip6_src,
 977     .name = "lookup-ip6-src",
 978     .vector_size = sizeof (u32),
 979     .format_trace = format_lookup_trace,
 980     .sibling_of = "ip6-lookup",
 981 };
 982 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_src_node, lookup_ip6_src)
 983
 984 always_inline uword
 985 lookup_dpo_mpls_inline (vlib_main_t * vm,
 986                        vlib_node_runtime_t * node,
 987                        vlib_frame_t * from_frame,
 988                        int table_from_interface)
 989 {
 990     u32 n_left_from, next_index, * from, * to_next;
 991     u32 thread_index = vlib_get_thread_index();
 992     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 993
 994     from = vlib_frame_vector_args (from_frame);
 995     n_left_from = from_frame->n_vectors;
 996
 997     next_index = node->cached_next_index;
 998
 999     while (n_left_from > 0)
1000     {
1001         u32 n_left_to_next;
1002
1003         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
1004
1005         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1006         /*   } */
1007
1008         while (n_left_from > 0 && n_left_to_next > 0)
1009         {
1010             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash0;
1011             const mpls_unicast_header_t * hdr0;
1012             const load_balance_t *lb0;
1013             const lookup_dpo_t * lkd0;
1014             const dpo_id_t *dpo0;
1015             vlib_buffer_t * b0;
1016
1017             bi0 = from[0];
1018             to_next[0] = bi0;
1019             from += 1;
1020             to_next += 1;
1021             n_left_from -= 1;
1022             n_left_to_next -= 1;
1023
1024             b0 = vlib_get_buffer (vm, bi0);
1025             hdr0 = vlib_buffer_get_current (b0);
1026
1027             /* dst lookup was done by mpls lookup */
1028             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1029             lkd0 = lookup_dpo_get(lkdi0);
1030
1031             /*
1032              * choose between a lookup using the fib index in the DPO
1033              * or getting the FIB index from the interface.
1034              */
1035             if (table_from_interface)
1036             {
1037                 fib_index0 =
1038                     mpls_fib_table_get_index_for_sw_if_index(
1039                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1040             }
1041             else
1042             {
1043                 fib_index0 = lkd0->lkd_fib_index;
1044             }
1045
1046             /* do lookup */
1047             lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
1048             lb0  = load_balance_get(lbi0);
1049             dpo0 = load_balance_get_bucket_i(lb0, 0);
1050
1051             next0 = dpo0->dpoi_next_node;
1052             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1053
1054
1055             if (MPLS_IS_REPLICATE & lbi0)
1056             {
1057                 next0 = mpls_lookup_to_replicate_edge;
1058                 vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
1059                     (lbi0 & ~MPLS_IS_REPLICATE);
1060             }
1061             else
1062             {
1063                 lb0 = load_balance_get(lbi0);
1064                 ASSERT (lb0->lb_n_buckets > 0);
1065                 ASSERT (is_pow2 (lb0->lb_n_buckets));
1066
1067                 if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
1068                 {
1069                     hash0 = vnet_buffer (b0)->ip.flow_hash =
1070                         mpls_compute_flow_hash(hdr0, lb0->lb_hash_config);
1071                     dpo0 = load_balance_get_fwd_bucket
1072                         (lb0,
1073                          (hash0 & (lb0->lb_n_buckets_minus_1)));
1074                 }
1075                 else
1076                 {
1077                     dpo0 = load_balance_get_bucket_i (lb0, 0);
1078                 }
1079                 next0 = dpo0->dpoi_next_node;
1080
1081                 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1082
1083                 vlib_increment_combined_counter
1084                     (cm, thread_index, lbi0, 1,
1085                      vlib_buffer_length_in_chain (vm, b0));
1086             }
1087
1088           vnet_buffer (b0)->mpls.ttl = ((char*)hdr0)[3];
1089             vnet_buffer (b0)->mpls.exp = (((char*)hdr0)[2] & 0xe) >> 1;
1090             vnet_buffer (b0)->mpls.first = 1;
1091             vlib_buffer_advance(b0, sizeof(*hdr0));
1092
1093             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1094             {
1095                 lookup_trace_t *tr = vlib_add_trace (vm, node,
1096                                                      b0, sizeof (*tr));
1097                 tr->fib_index = fib_index0;
1098                 tr->lbi = lbi0;
1099                 tr->hdr = *hdr0;
1100             }
1101
1102            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1103                                             n_left_to_next, bi0, next0);
1104         }
1105         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1106     }
1107     return from_frame->n_vectors;
1108 }
1109
1110 static u8 *
1111 format_lookup_mpls_trace (u8 * s, va_list * args)
1112 {
1113     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1114     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1115     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
1116     uword indent = format_get_indent (s);
1117     mpls_unicast_header_t hdr;
1118
1119     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
1120
1121     s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
1122                 format_white_space, indent,
1123                 t->fib_index,
1124                 format_mpls_header, hdr,
1125                 t->lbi);
1126     return s;
1127 }
1128
1129 always_inline uword
1130 lookup_mpls_dst (vlib_main_t * vm,
1131                 vlib_node_runtime_t * node,
1132                 vlib_frame_t * from_frame)
1133 {
1134     return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
1135 }
1136
1137 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
1138     .function = lookup_mpls_dst,
1139     .name = "lookup-mpls-dst",
1140     .vector_size = sizeof (u32),
1141     .sibling_of = "mpls-lookup",
1142     .format_trace = format_lookup_mpls_trace,
1143     .n_next_nodes = 0,
1144 };
1145 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_node, lookup_mpls_dst)
1146
1147 always_inline uword
1148 lookup_mpls_dst_itf (vlib_main_t * vm,
1149                     vlib_node_runtime_t * node,
1150                     vlib_frame_t * from_frame)
1151 {
1152     return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
1153 }
1154
1155 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
1156     .function = lookup_mpls_dst_itf,
1157     .name = "lookup-mpls-dst-itf",
1158     .vector_size = sizeof (u32),
1159     .sibling_of = "mpls-lookup",
1160     .format_trace = format_lookup_mpls_trace,
1161     .n_next_nodes = 0,
1162 };
1163 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf)
1164
1165 typedef enum lookup_ip_dst_mcast_next_t_ {
1166     LOOKUP_IP_DST_MCAST_NEXT_RPF,
1167     LOOKUP_IP_DST_MCAST_N_NEXT,
1168 } mfib_forward_lookup_next_t;
1169
1170 always_inline uword
1171 lookup_dpo_ip_dst_mcast_inline (vlib_main_t * vm,
1172                                 vlib_node_runtime_t * node,
1173                                 vlib_frame_t * from_frame,
1174                                 int is_v4)
1175 {
1176     u32 n_left_from, next_index, * from, * to_next;
1177
1178     from = vlib_frame_vector_args (from_frame);
1179     n_left_from = from_frame->n_vectors;
1180
1181     next_index = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1182
1183     while (n_left_from > 0)
1184     {
1185         u32 n_left_to_next;
1186
1187         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
1188
1189         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1190         /*   } */
1191
1192         while (n_left_from > 0 && n_left_to_next > 0)
1193         {
1194             u32 bi0, lkdi0, fib_index0,  next0;
1195             const lookup_dpo_t * lkd0;
1196             fib_node_index_t mfei0;
1197             vlib_buffer_t * b0;
1198
1199             bi0 = from[0];
1200             to_next[0] = bi0;
1201             from += 1;
1202             to_next += 1;
1203             n_left_from -= 1;
1204             n_left_to_next -= 1;
1205
1206             b0 = vlib_get_buffer (vm, bi0);
1207
1208             /* dst lookup was done by mpls lookup */
1209             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1210             lkd0 = lookup_dpo_get(lkdi0);
1211             fib_index0 = lkd0->lkd_fib_index;
1212             next0 = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1213
1214             if (is_v4)
1215             {
1216                 ip4_header_t * ip0;
1217
1218                 ip0 = vlib_buffer_get_current (b0);
1219                 mfei0 = ip4_mfib_table_lookup(ip4_mfib_get(fib_index0),
1220                                               &ip0->src_address,
1221                                               &ip0->dst_address,
1222                                               64);
1223                 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1224                 {
1225                     lookup_trace_t *tr = vlib_add_trace (vm, node,
1226                                                          b0, sizeof (*tr));
1227                     tr->fib_index = fib_index0;
1228                     tr->lbi = mfei0;
1229                     tr->addr.ip4 = ip0->dst_address;
1230                 }
1231             }
1232             else
1233             {
1234                 ip6_header_t * ip0;
1235
1236                 ip0 = vlib_buffer_get_current (b0);
1237                 mfei0 = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index0),
1238                                                &ip0->src_address,
1239                                                &ip0->dst_address);
1240                 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1241                 {
1242                     lookup_trace_t *tr = vlib_add_trace (vm, node,
1243                                                          b0, sizeof (*tr));
1244                     tr->fib_index = fib_index0;
1245                     tr->lbi = mfei0;
1246                     tr->addr.ip6 = ip0->dst_address;
1247                 }
1248             }
1249
1250             vnet_buffer (b0)->ip.adj_index[VLIB_TX] = mfei0;
1251
1252             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1253                                             n_left_to_next, bi0, next0);
1254         }
1255         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1256     }
1257     return from_frame->n_vectors;
1258 }
1259
1260 always_inline uword
1261 lookup_ip4_dst_mcast (vlib_main_t * vm,
1262                       vlib_node_runtime_t * node,
1263                       vlib_frame_t * from_frame)
1264 {
1265     return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 1));
1266 }
1267
1268 VLIB_REGISTER_NODE (lookup_ip4_dst_mcast_node) = {
1269     .function = lookup_ip4_dst_mcast,
1270     .name = "lookup-ip4-dst-mcast",
1271     .vector_size = sizeof (u32),
1272
1273     .format_trace = format_lookup_trace,
1274     .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1275     .next_nodes = {
1276         [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip4-mfib-forward-rpf",
1277     },
1278 };
1279 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_mcast_node,
1280                               lookup_ip4_dst_mcast)
1281
1282 always_inline uword
1283 lookup_ip6_dst_mcast (vlib_main_t * vm,
1284                       vlib_node_runtime_t * node,
1285                       vlib_frame_t * from_frame)
1286 {
1287     return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 0));
1288 }
1289
1290 VLIB_REGISTER_NODE (lookup_ip6_dst_mcast_node) = {
1291     .function = lookup_ip6_dst_mcast,
1292     .name = "lookup-ip6-dst-mcast",
1293     .vector_size = sizeof (u32),
1294
1295     .format_trace = format_lookup_trace,
1296     .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1297     .next_nodes = {
1298         [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip6-mfib-forward-rpf",
1299     },
1300 };
1301 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_mcast_node,
1302                               lookup_ip6_dst_mcast)
1303
1304 static void
1305 lookup_dpo_mem_show (void)
1306 {
1307     fib_show_memory_usage("Lookup",
1308                           pool_elts(lookup_dpo_pool),
1309                           pool_len(lookup_dpo_pool),
1310                           sizeof(lookup_dpo_t));
1311 }
1312
1313 const static dpo_vft_t lkd_vft = {
1314     .dv_lock = lookup_dpo_lock,
1315     .dv_unlock = lookup_dpo_unlock,
1316     .dv_format = format_lookup_dpo,
1317 };
1318 const static dpo_vft_t lkd_vft_w_mem_show = {
1319     .dv_lock = lookup_dpo_lock,
1320     .dv_unlock = lookup_dpo_unlock,
1321     .dv_format = format_lookup_dpo,
1322     .dv_mem_show = lookup_dpo_mem_show,
1323 };
1324
1325 const static char* const lookup_src_ip4_nodes[] =
1326 {
1327     "lookup-ip4-src",
1328     NULL,
1329 };
1330 const static char* const lookup_src_ip6_nodes[] =
1331 {
1332     "lookup-ip6-src",
1333     NULL,
1334 };
1335 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
1336 {
1337     [DPO_PROTO_IP4]  = lookup_src_ip4_nodes,
1338     [DPO_PROTO_IP6]  = lookup_src_ip6_nodes,
1339     [DPO_PROTO_MPLS] = NULL,
1340 };
1341
1342 const static char* const lookup_dst_ip4_nodes[] =
1343 {
1344     "lookup-ip4-dst",
1345     NULL,
1346 };
1347 const static char* const lookup_dst_ip6_nodes[] =
1348 {
1349     "lookup-ip6-dst",
1350     NULL,
1351 };
1352 const static char* const lookup_dst_mpls_nodes[] =
1353 {
1354     "lookup-mpls-dst",
1355     NULL,
1356 };
1357 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
1358 {
1359     [DPO_PROTO_IP4]  = lookup_dst_ip4_nodes,
1360     [DPO_PROTO_IP6]  = lookup_dst_ip6_nodes,
1361     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
1362 };
1363
1364 const static char* const lookup_dst_mcast_ip4_nodes[] =
1365 {
1366     "lookup-ip4-dst-mcast",
1367     NULL,
1368 };
1369 const static char* const lookup_dst_mcast_ip6_nodes[] =
1370 {
1371     "lookup-ip6-dst-mcast",
1372     NULL,
1373 };
1374 const static char* const * const lookup_dst_mcast_nodes[DPO_PROTO_NUM] =
1375 {
1376     [DPO_PROTO_IP4]  = lookup_dst_mcast_ip4_nodes,
1377     [DPO_PROTO_IP6]  = lookup_dst_mcast_ip6_nodes,
1378 };
1379
1380 const static char* const lookup_dst_from_interface_ip4_nodes[] =
1381 {
1382     "lookup-ip4-dst-itf",
1383     NULL,
1384 };
1385 const static char* const lookup_dst_from_interface_ip6_nodes[] =
1386 {
1387     "lookup-ip6-dst-itf",
1388     NULL,
1389 };
1390 const static char* const lookup_dst_from_interface_mpls_nodes[] =
1391 {
1392     "lookup-mpls-dst-itf",
1393     NULL,
1394 };
1395 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
1396 {
1397     [DPO_PROTO_IP4]  = lookup_dst_from_interface_ip4_nodes,
1398     [DPO_PROTO_IP6]  = lookup_dst_from_interface_ip6_nodes,
1399     [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
1400 };
1401
1402
1403 void
1404 lookup_dpo_module_init (void)
1405 {
1406     dpo_register(DPO_LOOKUP, &lkd_vft_w_mem_show, NULL);
1407
1408     /*
1409      * There are various sorts of lookup; src or dst addr v4 /v6 etc.
1410      * there isn't an object type for each (there is only the lookup_dpo_t),
1411      * but, for performance reasons, there is a data plane function, and hence
1412      * VLIB node for each. VLIB graph node construction is based on DPO types
1413      * so we create sub-types.
1414      */
1415     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
1416         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
1417     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
1418         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
1419     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST] =
1420         dpo_register_new_type(&lkd_vft, lookup_dst_mcast_nodes);
1421     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
1422         dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes);
1423 }