src/vnet/dpo/lookup_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/lookup_dpo.h>
  18 #include <vnet/dpo/load_balance_map.h>
  19 #include <vnet/mpls/mpls_lookup.h>
  20 #include <vnet/fib/fib_table.h>
  21 #include <vnet/fib/ip4_fib.h>
  22 #include <vnet/fib/ip6_fib.h>
  23 #include <vnet/fib/mpls_fib.h>
  24 #include <vnet/mfib/mfib_table.h>
  25 #include <vnet/mfib/ip4_mfib.h>
  26 #include <vnet/mfib/ip6_mfib.h>
  27
  28 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
  29 static const char *const lookup_cast_names[] = LOOKUP_CASTS;
  30
  31 /**
  32  * @brief Enumeration of the lookup subtypes
  33  */
  34 typedef enum lookup_sub_type_t_
  35 {
  36     LOOKUP_SUB_TYPE_SRC,
  37     LOOKUP_SUB_TYPE_DST,
  38     LOOKUP_SUB_TYPE_DST_MCAST,
  39     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
  40 } lookup_sub_type_t;
  41 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
  42
  43 #define FOR_EACH_LOOKUP_SUB_TYPE(_st)                                   \
  44     for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
  45
  46 /**
  47  * @brief pool of all MPLS Label DPOs
  48  */
  49 lookup_dpo_t *lookup_dpo_pool;
  50
  51 /**
  52  * @brief An array of registered DPO type values for the sub-types
  53  */
  54 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
  55
  56 static lookup_dpo_t *
  57 lookup_dpo_alloc (void)
  58 {
  59     lookup_dpo_t *lkd;
  60
  61     pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
  62
  63     return (lkd);
  64 }
  65
  66 static index_t
  67 lookup_dpo_get_index (lookup_dpo_t *lkd)
  68 {
  69     return (lkd - lookup_dpo_pool);
  70 }
  71
  72 static void
  73 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
  74                           dpo_proto_t proto,
  75                           lookup_cast_t cast,
  76                           lookup_input_t input,
  77                           lookup_table_t table_config,
  78                           dpo_id_t *dpo)
  79 {
  80     lookup_dpo_t *lkd;
  81     dpo_type_t type;
  82
  83     lkd = lookup_dpo_alloc();
  84     lkd->lkd_fib_index = fib_index;
  85     lkd->lkd_proto = proto;
  86     lkd->lkd_input = input;
  87     lkd->lkd_table = table_config;
  88     lkd->lkd_cast  = cast;
  89
  90     /*
  91      * use the input type to select the lookup sub-type
  92      */
  93     type = 0;
  94
  95     switch (input)
  96     {
  97     case LOOKUP_INPUT_SRC_ADDR:
  98         type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
  99         break;
 100     case LOOKUP_INPUT_DST_ADDR:
 101         switch (table_config)
 102         {
 103         case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
 104             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
 105             break;
 106         case LOOKUP_TABLE_FROM_CONFIG:
 107             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
 108             break;
 109         }
 110         if (LOOKUP_MULTICAST == cast)
 111         {
 112             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST];
 113         }
 114     }
 115
 116     if (0 == type)
 117     {
 118         dpo_reset(dpo);
 119     }
 120     else
 121     {
 122         dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
 123     }
 124 }
 125
 126 void
 127 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
 128                                     dpo_proto_t proto,
 129                                     lookup_cast_t cast,
 130                                     lookup_input_t input,
 131                                     lookup_table_t table_config,
 132                                     dpo_id_t *dpo)
 133 {
 134     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 135     {
 136         if (LOOKUP_UNICAST == cast)
 137         {
 138             fib_table_lock(fib_index, dpo_proto_to_fib(proto));
 139         }
 140         else
 141         {
 142             mfib_table_lock(fib_index, dpo_proto_to_fib(proto));
 143         }
 144     }
 145     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 146 }
 147
 148 void
 149 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 150                                    dpo_proto_t proto,
 151                                    lookup_cast_t cast,
 152                                    lookup_input_t input,
 153                                    lookup_table_t table_config,
 154                                    dpo_id_t *dpo)
 155 {
 156     fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
 157
 158     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 159     {
 160         if (LOOKUP_UNICAST == cast)
 161         {
 162             fib_index =
 163                 fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 164                                                   table_id);
 165         }
 166         else
 167         {
 168             fib_index =
 169                 mfib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 170                                                    table_id);
 171         }
 172     }
 173
 174     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
 175     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 176 }
 177
 178 u8*
 179 format_lookup_dpo (u8 *s, va_list *args)
 180 {
 181     index_t index = va_arg (*args, index_t);
 182     lookup_dpo_t *lkd;
 183
 184     lkd = lookup_dpo_get(index);
 185
 186     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
 187     {
 188         s = format(s, "%s,%s lookup in interface's %U table",
 189                    lookup_input_names[lkd->lkd_input],
 190                    lookup_cast_names[lkd->lkd_cast],
 191                    format_dpo_proto, lkd->lkd_proto);
 192     }
 193     else
 194     {
 195         if (LOOKUP_UNICAST == lkd->lkd_cast)
 196         {
 197             s = format(s, "%s,%s lookup in %U",
 198                        lookup_input_names[lkd->lkd_input],
 199                        lookup_cast_names[lkd->lkd_cast],
 200                        format_fib_table_name, lkd->lkd_fib_index,
 201                        dpo_proto_to_fib(lkd->lkd_proto));
 202         }
 203         else
 204         {
 205             s = format(s, "%s,%s lookup in %U",
 206                        lookup_input_names[lkd->lkd_input],
 207                        lookup_cast_names[lkd->lkd_cast],
 208                        format_mfib_table_name, lkd->lkd_fib_index,
 209                        dpo_proto_to_fib(lkd->lkd_proto));
 210         }
 211     }
 212     return (s);
 213 }
 214
 215 static void
 216 lookup_dpo_lock (dpo_id_t *dpo)
 217 {
 218     lookup_dpo_t *lkd;
 219
 220     lkd = lookup_dpo_get(dpo->dpoi_index);
 221
 222     lkd->lkd_locks++;
 223 }
 224
 225 static void
 226 lookup_dpo_unlock (dpo_id_t *dpo)
 227 {
 228     lookup_dpo_t *lkd;
 229
 230     lkd = lookup_dpo_get(dpo->dpoi_index);
 231
 232     lkd->lkd_locks--;
 233
 234     if (0 == lkd->lkd_locks)
 235     {
 236         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
 237         {
 238             if (LOOKUP_UNICAST == lkd->lkd_cast)
 239             {
 240                 fib_table_unlock(lkd->lkd_fib_index,
 241                                  dpo_proto_to_fib(lkd->lkd_proto));
 242             }
 243             else
 244             {
 245                 mfib_table_unlock(lkd->lkd_fib_index,
 246                                   dpo_proto_to_fib(lkd->lkd_proto));
 247             }
 248         }
 249         pool_put(lookup_dpo_pool, lkd);
 250     }
 251 }
 252
 253 always_inline void
 254 ip4_src_fib_lookup_one (u32 src_fib_index0,
 255                         const ip4_address_t * addr0,
 256                         u32 * src_adj_index0)
 257 {
 258     ip4_fib_mtrie_leaf_t leaf0;
 259     ip4_fib_mtrie_t * mtrie0;
 260
 261     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 262
 263     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 264     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 265     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 266
 267     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 268 }
 269
 270 always_inline void
 271 ip4_src_fib_lookup_two (u32 src_fib_index0,
 272                         u32 src_fib_index1,
 273                         const ip4_address_t * addr0,
 274                         const ip4_address_t * addr1,
 275                         u32 * src_adj_index0,
 276                         u32 * src_adj_index1)
 277 {
 278     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 279     ip4_fib_mtrie_t * mtrie0, * mtrie1;
 280
 281     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 282     mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
 283
 284     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 285     leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, addr1);
 286
 287     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 288     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
 289
 290     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 291     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
 292
 293     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 294     src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 295 }
 296
 297 /**
 298  * @brief Lookup trace  data
 299  */
 300 typedef struct lookup_trace_t_
 301 {
 302     union {
 303         ip46_address_t addr;
 304         mpls_unicast_header_t hdr;
 305     };
 306     fib_node_index_t fib_index;
 307     index_t lbi;
 308 } lookup_trace_t;
 309
 310
 311 always_inline uword
 312 lookup_dpo_ip4_inline (vlib_main_t * vm,
 313                        vlib_node_runtime_t * node,
 314                        vlib_frame_t * from_frame,
 315                        int input_src_addr,
 316                        int table_from_interface)
 317 {
 318     u32 n_left_from, next_index, * from, * to_next;
 319     u32 thread_index = vlib_get_thread_index();
 320     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 321
 322     from = vlib_frame_vector_args (from_frame);
 323     n_left_from = from_frame->n_vectors;
 324
 325     next_index = node->cached_next_index;
 326
 327     while (n_left_from > 0)
 328     {
 329         u32 n_left_to_next;
 330
 331         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 332
 333         while (n_left_from >= 4 && n_left_to_next > 2)
 334         {
 335             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 336             flow_hash_config_t flow_hash_config0;
 337             const ip4_address_t *input_addr0;
 338             const load_balance_t *lb0;
 339             const lookup_dpo_t * lkd0;
 340             const ip4_header_t * ip0;
 341             const dpo_id_t *dpo0;
 342             vlib_buffer_t * b0;
 343             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 344             flow_hash_config_t flow_hash_config1;
 345             const ip4_address_t *input_addr1;
 346             const load_balance_t *lb1;
 347             const lookup_dpo_t * lkd1;
 348             const ip4_header_t * ip1;
 349             const dpo_id_t *dpo1;
 350             vlib_buffer_t * b1;
 351
 352             /* Prefetch next iteration. */
 353             {
 354                 vlib_buffer_t * p2, * p3;
 355
 356                 p2 = vlib_get_buffer (vm, from[2]);
 357                 p3 = vlib_get_buffer (vm, from[3]);
 358
 359                 vlib_prefetch_buffer_header (p2, LOAD);
 360                 vlib_prefetch_buffer_header (p3, LOAD);
 361
 362                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 363                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 364             }
 365
 366             bi0 = from[0];
 367             to_next[0] = bi0;
 368             bi1 = from[1];
 369             to_next[1] = bi1;
 370             from += 2;
 371             to_next += 2;
 372             n_left_from -= 2;
 373             n_left_to_next -= 2;
 374
 375             b0 = vlib_get_buffer (vm, bi0);
 376             ip0 = vlib_buffer_get_current (b0);
 377             b1 = vlib_get_buffer (vm, bi1);
 378             ip1 = vlib_buffer_get_current (b1);
 379
 380             /* dst lookup was done by ip4 lookup */
 381             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 382             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 383             lkd0 = lookup_dpo_get(lkdi0);
 384             lkd1 = lookup_dpo_get(lkdi1);
 385
 386             /*
 387              * choose between a lookup using the fib index in the DPO
 388              * or getting the FIB index from the interface.
 389              */
 390             if (table_from_interface)
 391             {
 392                 fib_index0 =
 393                     ip4_fib_table_get_index_for_sw_if_index(
 394                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 395                 fib_index1 =
 396                     ip4_fib_table_get_index_for_sw_if_index(
 397                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 398             }
 399             else
 400             {
 401                 fib_index0 = lkd0->lkd_fib_index;
 402                 fib_index1 = lkd1->lkd_fib_index;
 403             }
 404
 405             /*
 406              * choose between a source or destination address lookup in the table
 407              */
 408             if (input_src_addr)
 409             {
 410                 input_addr0 = &ip0->src_address;
 411                 input_addr1 = &ip1->src_address;
 412             }
 413             else
 414             {
 415                 input_addr0 = &ip0->dst_address;
 416                 input_addr1 = &ip1->dst_address;
 417             }
 418
 419             /* do lookup */
 420             ip4_src_fib_lookup_two (fib_index0, fib_index1,
 421                                     input_addr0, input_addr1,
 422                                     &lbi0, &lbi1);
 423             lb0 = load_balance_get(lbi0);
 424             lb1 = load_balance_get(lbi1);
 425
 426             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 427             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 428
 429             /* Use flow hash to compute multipath adjacency. */
 430             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 431             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 432
 433             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 434             {
 435                 flow_hash_config0 = lb0->lb_hash_config;
 436                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 437                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 438             }
 439
 440             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 441             {
 442                 flow_hash_config1 = lb1->lb_hash_config;
 443                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 444                     ip4_compute_flow_hash (ip1, flow_hash_config1);
 445             }
 446
 447             dpo0 = load_balance_get_bucket_i(lb0,
 448                                              (hash_c0 &
 449                                               (lb0->lb_n_buckets_minus_1)));
 450             dpo1 = load_balance_get_bucket_i(lb1,
 451                                              (hash_c1 &
 452                                               (lb1->lb_n_buckets_minus_1)));
 453
 454             next0 = dpo0->dpoi_next_node;
 455             next1 = dpo1->dpoi_next_node;
 456             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 457             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 458
 459             vlib_increment_combined_counter
 460                 (cm, thread_index, lbi0, 1,
 461                  vlib_buffer_length_in_chain (vm, b0));
 462             vlib_increment_combined_counter
 463                 (cm, thread_index, lbi1, 1,
 464                  vlib_buffer_length_in_chain (vm, b1));
 465
 466             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 467             {
 468                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 469                                                      b0, sizeof (*tr));
 470                 tr->fib_index = fib_index0;
 471                 tr->lbi = lbi0;
 472                 tr->addr.ip4 = *input_addr0;
 473             }
 474             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 475             {
 476                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 477                                                      b1, sizeof (*tr));
 478                 tr->fib_index = fib_index1;
 479                 tr->lbi = lbi1;
 480                 tr->addr.ip4 = *input_addr1;
 481             }
 482
 483             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 484                                              to_next, n_left_to_next,
 485                                              bi0, bi1, next0, next1);
 486         }
 487
 488         while (n_left_from > 0 && n_left_to_next > 0)
 489         {
 490             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 491             flow_hash_config_t flow_hash_config0;
 492             const ip4_address_t *input_addr;
 493             const load_balance_t *lb0;
 494             const lookup_dpo_t * lkd0;
 495             const ip4_header_t * ip0;
 496             const dpo_id_t *dpo0;
 497             vlib_buffer_t * b0;
 498
 499             bi0 = from[0];
 500             to_next[0] = bi0;
 501             from += 1;
 502             to_next += 1;
 503             n_left_from -= 1;
 504             n_left_to_next -= 1;
 505
 506             b0 = vlib_get_buffer (vm, bi0);
 507             ip0 = vlib_buffer_get_current (b0);
 508
 509             /* dst lookup was done by ip4 lookup */
 510             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 511             lkd0 = lookup_dpo_get(lkdi0);
 512
 513             /*
 514              * choose between a lookup using the fib index in the DPO
 515              * or getting the FIB index from the interface.
 516              */
 517             if (table_from_interface)
 518             {
 519                 fib_index0 =
 520                     ip4_fib_table_get_index_for_sw_if_index(
 521                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 522             }
 523             else
 524             {
 525                 fib_index0 = lkd0->lkd_fib_index;
 526             }
 527
 528             /*
 529              * choose between a source or destination address lookup in the table
 530              */
 531             if (input_src_addr)
 532             {
 533                 input_addr = &ip0->src_address;
 534             }
 535             else
 536             {
 537                 input_addr = &ip0->dst_address;
 538             }
 539
 540             /* do lookup */
 541             ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
 542             lb0 = load_balance_get(lbi0);
 543
 544             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 545
 546             /* Use flow hash to compute multipath adjacency. */
 547             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 548
 549             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 550             {
 551                 flow_hash_config0 = lb0->lb_hash_config;
 552                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 553                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 554             }
 555
 556             dpo0 = load_balance_get_bucket_i(lb0,
 557                                              (hash_c0 &
 558                                               (lb0->lb_n_buckets_minus_1)));
 559
 560             next0 = dpo0->dpoi_next_node;
 561             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 562
 563             vlib_increment_combined_counter
 564                 (cm, thread_index, lbi0, 1,
 565                  vlib_buffer_length_in_chain (vm, b0));
 566
 567             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 568             {
 569                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 570                                                      b0, sizeof (*tr));
 571                 tr->fib_index = fib_index0;
 572                 tr->lbi = lbi0;
 573                 tr->addr.ip4 = *input_addr;
 574             }
 575
 576             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 577                                             n_left_to_next, bi0, next0);
 578         }
 579         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 580     }
 581     return from_frame->n_vectors;
 582 }
 583
 584 static u8 *
 585 format_lookup_trace (u8 * s, va_list * args)
 586 {
 587     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 588     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 589     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 590     uword indent = format_get_indent (s);
 591     s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
 592                 format_white_space, indent,
 593                 t->fib_index,
 594                 format_ip46_address, &t->addr, IP46_TYPE_ANY,
 595                 t->lbi);
 596     return s;
 597 }
 598
 599 always_inline uword
 600 lookup_ip4_dst (vlib_main_t * vm,
 601                 vlib_node_runtime_t * node,
 602                 vlib_frame_t * from_frame)
 603 {
 604     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
 605 }
 606
 607 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
 608     .function = lookup_ip4_dst,
 609     .name = "lookup-ip4-dst",
 610     .vector_size = sizeof (u32),
 611     .sibling_of = "ip4-lookup",
 612     .format_trace = format_lookup_trace,
 613 };
 614 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_node, lookup_ip4_dst)
 615
 616 always_inline uword
 617 lookup_ip4_dst_itf (vlib_main_t * vm,
 618                     vlib_node_runtime_t * node,
 619                     vlib_frame_t * from_frame)
 620 {
 621     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
 622 }
 623
 624 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
 625     .function = lookup_ip4_dst_itf,
 626     .name = "lookup-ip4-dst-itf",
 627     .vector_size = sizeof (u32),
 628     .sibling_of = "ip4-lookup",
 629     .format_trace = format_lookup_trace,
 630 };
 631 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_itf_node, lookup_ip4_dst_itf)
 632
 633 always_inline uword
 634 lookup_ip4_src (vlib_main_t * vm,
 635                 vlib_node_runtime_t * node,
 636                 vlib_frame_t * from_frame)
 637 {
 638     return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
 639 }
 640
 641 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
 642     .function = lookup_ip4_src,
 643     .name = "lookup-ip4-src",
 644     .vector_size = sizeof (u32),
 645     .format_trace = format_lookup_trace,
 646     .sibling_of = "ip4-lookup",
 647 };
 648 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_src_node, lookup_ip4_src)
 649
 650 always_inline uword
 651 lookup_dpo_ip6_inline (vlib_main_t * vm,
 652                        vlib_node_runtime_t * node,
 653                        vlib_frame_t * from_frame,
 654                        int input_src_addr,
 655                        int table_from_interface)
 656 {
 657     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 658     u32 n_left_from, next_index, * from, * to_next;
 659     u32 thread_index = vlib_get_thread_index();
 660
 661     from = vlib_frame_vector_args (from_frame);
 662     n_left_from = from_frame->n_vectors;
 663
 664     next_index = node->cached_next_index;
 665
 666     while (n_left_from > 0)
 667     {
 668         u32 n_left_to_next;
 669
 670         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 671
 672         while (n_left_from >= 4 && n_left_to_next > 2)
 673         {
 674             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 675             flow_hash_config_t flow_hash_config0;
 676             const ip6_address_t *input_addr0;
 677             const load_balance_t *lb0;
 678             const lookup_dpo_t * lkd0;
 679             const ip6_header_t * ip0;
 680             const dpo_id_t *dpo0;
 681             vlib_buffer_t * b0;
 682             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 683             flow_hash_config_t flow_hash_config1;
 684             const ip6_address_t *input_addr1;
 685             const load_balance_t *lb1;
 686             const lookup_dpo_t * lkd1;
 687             const ip6_header_t * ip1;
 688             const dpo_id_t *dpo1;
 689             vlib_buffer_t * b1;
 690
 691             /* Prefetch next iteration. */
 692             {
 693                 vlib_buffer_t * p2, * p3;
 694
 695                 p2 = vlib_get_buffer (vm, from[2]);
 696                 p3 = vlib_get_buffer (vm, from[3]);
 697
 698                 vlib_prefetch_buffer_header (p2, LOAD);
 699                 vlib_prefetch_buffer_header (p3, LOAD);
 700
 701                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 702                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 703             }
 704
 705             bi0 = from[0];
 706             to_next[0] = bi0;
 707             bi1 = from[1];
 708             to_next[1] = bi1;
 709             from += 2;
 710             to_next += 2;
 711             n_left_from -= 2;
 712             n_left_to_next -= 2;
 713
 714             b0 = vlib_get_buffer (vm, bi0);
 715             ip0 = vlib_buffer_get_current (b0);
 716             b1 = vlib_get_buffer (vm, bi1);
 717             ip1 = vlib_buffer_get_current (b1);
 718
 719             /* dst lookup was done by ip6 lookup */
 720             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 721             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 722             lkd0 = lookup_dpo_get(lkdi0);
 723             lkd1 = lookup_dpo_get(lkdi1);
 724
 725             /*
 726              * choose between a lookup using the fib index in the DPO
 727              * or getting the FIB index from the interface.
 728              */
 729             if (table_from_interface)
 730             {
 731                 fib_index0 =
 732                     ip6_fib_table_get_index_for_sw_if_index(
 733                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 734                 fib_index1 =
 735                     ip6_fib_table_get_index_for_sw_if_index(
 736                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 737             }
 738             else
 739             {
 740                 fib_index0 = lkd0->lkd_fib_index;
 741                 fib_index1 = lkd1->lkd_fib_index;
 742             }
 743
 744             /*
 745              * choose between a source or destination address lookup in the table
 746              */
 747             if (input_src_addr)
 748             {
 749                 input_addr0 = &ip0->src_address;
 750                 input_addr1 = &ip1->src_address;
 751             }
 752             else
 753             {
 754                 input_addr0 = &ip0->dst_address;
 755                 input_addr1 = &ip1->dst_address;
 756             }
 757
 758             /* do src lookup */
 759             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 760                                                fib_index0,
 761                                                input_addr0);
 762             lbi1 = ip6_fib_table_fwding_lookup(&ip6_main,
 763                                                fib_index1,
 764                                                input_addr1);
 765             lb0 = load_balance_get(lbi0);
 766             lb1 = load_balance_get(lbi1);
 767
 768             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 769             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 770
 771             /* Use flow hash to compute multipath adjacency. */
 772             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 773             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 774
 775             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 776             {
 777                 flow_hash_config0 = lb0->lb_hash_config;
 778                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 779                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 780             }
 781
 782             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 783             {
 784                 flow_hash_config1 = lb1->lb_hash_config;
 785                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 786                     ip6_compute_flow_hash (ip1, flow_hash_config1);
 787             }
 788
 789             dpo0 = load_balance_get_bucket_i(lb0,
 790                                              (hash_c0 &
 791                                               (lb0->lb_n_buckets_minus_1)));
 792             dpo1 = load_balance_get_bucket_i(lb1,
 793                                              (hash_c1 &
 794                                               (lb1->lb_n_buckets_minus_1)));
 795
 796             next0 = dpo0->dpoi_next_node;
 797             next1 = dpo1->dpoi_next_node;
 798             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 799             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 800
 801             vlib_increment_combined_counter
 802                 (cm, thread_index, lbi0, 1,
 803                  vlib_buffer_length_in_chain (vm, b0));
 804             vlib_increment_combined_counter
 805                 (cm, thread_index, lbi1, 1,
 806                  vlib_buffer_length_in_chain (vm, b1));
 807
 808             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 809             {
 810                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 811                                                      b0, sizeof (*tr));
 812                 tr->fib_index = fib_index0;
 813                 tr->lbi = lbi0;
 814                 tr->addr.ip6 = *input_addr0;
 815             }
 816             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 817             {
 818                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 819                                                      b1, sizeof (*tr));
 820                 tr->fib_index = fib_index1;
 821                 tr->lbi = lbi1;
 822                 tr->addr.ip6 = *input_addr1;
 823             }
 824             vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
 825                                             n_left_to_next, bi0, bi1,
 826                                             next0, next1);
 827         }
 828         while (n_left_from > 0 && n_left_to_next > 0)
 829         {
 830             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 831             flow_hash_config_t flow_hash_config0;
 832             const ip6_address_t *input_addr0;
 833             const load_balance_t *lb0;
 834             const lookup_dpo_t * lkd0;
 835             const ip6_header_t * ip0;
 836             const dpo_id_t *dpo0;
 837             vlib_buffer_t * b0;
 838
 839             bi0 = from[0];
 840             to_next[0] = bi0;
 841             from += 1;
 842             to_next += 1;
 843             n_left_from -= 1;
 844             n_left_to_next -= 1;
 845
 846             b0 = vlib_get_buffer (vm, bi0);
 847             ip0 = vlib_buffer_get_current (b0);
 848
 849             /* dst lookup was done by ip6 lookup */
 850             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 851             lkd0 = lookup_dpo_get(lkdi0);
 852
 853             /*
 854              * choose between a lookup using the fib index in the DPO
 855              * or getting the FIB index from the interface.
 856              */
 857             if (table_from_interface)
 858             {
 859                 fib_index0 =
 860                     ip6_fib_table_get_index_for_sw_if_index(
 861                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 862             }
 863             else
 864             {
 865                 fib_index0 = lkd0->lkd_fib_index;
 866             }
 867
 868             /*
 869              * choose between a source or destination address lookup in the table
 870              */
 871             if (input_src_addr)
 872             {
 873                 input_addr0 = &ip0->src_address;
 874             }
 875             else
 876             {
 877                 input_addr0 = &ip0->dst_address;
 878             }
 879
 880             /* do src lookup */
 881             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 882                                                fib_index0,
 883                                                input_addr0);
 884             lb0 = load_balance_get(lbi0);
 885
 886             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 887
 888             /* Use flow hash to compute multipath adjacency. */
 889             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 890
 891             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 892             {
 893                 flow_hash_config0 = lb0->lb_hash_config;
 894                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 895                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 896             }
 897
 898             dpo0 = load_balance_get_bucket_i(lb0,
 899                                              (hash_c0 &
 900                                               (lb0->lb_n_buckets_minus_1)));
 901
 902             next0 = dpo0->dpoi_next_node;
 903             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 904
 905             vlib_increment_combined_counter
 906                 (cm, thread_index, lbi0, 1,
 907                  vlib_buffer_length_in_chain (vm, b0));
 908
 909             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 910             {
 911                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 912                                                      b0, sizeof (*tr));
 913                 tr->fib_index = fib_index0;
 914                 tr->lbi = lbi0;
 915                 tr->addr.ip6 = *input_addr0;
 916             }
 917             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 918                                             n_left_to_next, bi0, next0);
 919         }
 920         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 921     }
 922     return from_frame->n_vectors;
 923 }
 924
 925 always_inline uword
 926 lookup_ip6_dst (vlib_main_t * vm,
 927                 vlib_node_runtime_t * node,
 928                 vlib_frame_t * from_frame)
 929 {
 930     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 0));
 931 }
 932
 933 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
 934     .function = lookup_ip6_dst,
 935     .name = "lookup-ip6-dst",
 936     .vector_size = sizeof (u32),
 937     .format_trace = format_lookup_trace,
 938     .sibling_of = "ip6-lookup",
 939 };
 940 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_node, lookup_ip6_dst)
 941
 942 always_inline uword
 943 lookup_ip6_dst_itf (vlib_main_t * vm,
 944                     vlib_node_runtime_t * node,
 945                     vlib_frame_t * from_frame)
 946 {
 947     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 1));
 948 }
 949
 950 VLIB_REGISTER_NODE (lookup_ip6_dst_itf_node) = {
 951     .function = lookup_ip6_dst_itf,
 952     .name = "lookup-ip6-dst-itf",
 953     .vector_size = sizeof (u32),
 954     .format_trace = format_lookup_trace,
 955     .sibling_of = "ip6-lookup",
 956 };
 957 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_itf_node, lookup_ip6_dst_itf)
 958
 959 always_inline uword
 960 lookup_ip6_src (vlib_main_t * vm,
 961                 vlib_node_runtime_t * node,
 962                 vlib_frame_t * from_frame)
 963 {
 964     return (lookup_dpo_ip6_inline(vm, node, from_frame, 1, 0));
 965 }
 966
 967 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
 968     .function = lookup_ip6_src,
 969     .name = "lookup-ip6-src",
 970     .vector_size = sizeof (u32),
 971     .format_trace = format_lookup_trace,
 972     .sibling_of = "ip6-lookup",
 973 };
 974 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_src_node, lookup_ip6_src)
 975
 976 always_inline uword
 977 lookup_dpo_mpls_inline (vlib_main_t * vm,
 978                        vlib_node_runtime_t * node,
 979                        vlib_frame_t * from_frame,
 980                        int table_from_interface)
 981 {
 982     u32 n_left_from, next_index, * from, * to_next;
 983     u32 thread_index = vlib_get_thread_index();
 984     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 985
 986     from = vlib_frame_vector_args (from_frame);
 987     n_left_from = from_frame->n_vectors;
 988
 989     next_index = node->cached_next_index;
 990
 991     while (n_left_from > 0)
 992     {
 993         u32 n_left_to_next;
 994
 995         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 996
 997         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
 998         /*   } */
 999
1000         while (n_left_from > 0 && n_left_to_next > 0)
1001         {
1002             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash0;
1003             const mpls_unicast_header_t * hdr0;
1004             const load_balance_t *lb0;
1005             const lookup_dpo_t * lkd0;
1006             const dpo_id_t *dpo0;
1007             vlib_buffer_t * b0;
1008
1009             bi0 = from[0];
1010             to_next[0] = bi0;
1011             from += 1;
1012             to_next += 1;
1013             n_left_from -= 1;
1014             n_left_to_next -= 1;
1015
1016             b0 = vlib_get_buffer (vm, bi0);
1017             hdr0 = vlib_buffer_get_current (b0);
1018
1019             /* dst lookup was done by mpls lookup */
1020             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1021             lkd0 = lookup_dpo_get(lkdi0);
1022
1023             /*
1024              * choose between a lookup using the fib index in the DPO
1025              * or getting the FIB index from the interface.
1026              */
1027             if (table_from_interface)
1028             {
1029                 fib_index0 =
1030                     mpls_fib_table_get_index_for_sw_if_index(
1031                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1032             }
1033             else
1034             {
1035                 fib_index0 = lkd0->lkd_fib_index;
1036             }
1037
1038             /* do lookup */
1039             lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
1040             lb0  = load_balance_get(lbi0);
1041             dpo0 = load_balance_get_bucket_i(lb0, 0);
1042
1043             next0 = dpo0->dpoi_next_node;
1044             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1045
1046
1047             if (MPLS_IS_REPLICATE & lbi0)
1048             {
1049                 next0 = mpls_lookup_to_replicate_edge;
1050                 vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
1051                     (lbi0 & ~MPLS_IS_REPLICATE);
1052             }
1053             else
1054             {
1055                 lb0 = load_balance_get(lbi0);
1056                 ASSERT (lb0->lb_n_buckets > 0);
1057                 ASSERT (is_pow2 (lb0->lb_n_buckets));
1058
1059                 if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
1060                 {
1061                     hash0 = vnet_buffer (b0)->ip.flow_hash =
1062                         mpls_compute_flow_hash(hdr0, lb0->lb_hash_config);
1063                     dpo0 = load_balance_get_fwd_bucket
1064                         (lb0,
1065                          (hash0 & (lb0->lb_n_buckets_minus_1)));
1066                 }
1067                 else
1068                 {
1069                     dpo0 = load_balance_get_bucket_i (lb0, 0);
1070                 }
1071                 next0 = dpo0->dpoi_next_node;
1072
1073                 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1074
1075                 vlib_increment_combined_counter
1076                     (cm, thread_index, lbi0, 1,
1077                      vlib_buffer_length_in_chain (vm, b0));
1078             }
1079
1080           vnet_buffer (b0)->mpls.ttl = ((char*)hdr0)[3];
1081             vnet_buffer (b0)->mpls.exp = (((char*)hdr0)[2] & 0xe) >> 1;
1082             vnet_buffer (b0)->mpls.first = 1;
1083             vlib_buffer_advance(b0, sizeof(*hdr0));
1084
1085             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1086             {
1087                 lookup_trace_t *tr = vlib_add_trace (vm, node,
1088                                                      b0, sizeof (*tr));
1089                 tr->fib_index = fib_index0;
1090                 tr->lbi = lbi0;
1091                 tr->hdr = *hdr0;
1092             }
1093
1094            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1095                                             n_left_to_next, bi0, next0);
1096         }
1097         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1098     }
1099     return from_frame->n_vectors;
1100 }
1101
1102 static u8 *
1103 format_lookup_mpls_trace (u8 * s, va_list * args)
1104 {
1105     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1106     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1107     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
1108     uword indent = format_get_indent (s);
1109     mpls_unicast_header_t hdr;
1110
1111     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
1112
1113     s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
1114                 format_white_space, indent,
1115                 t->fib_index,
1116                 format_mpls_header, hdr,
1117                 t->lbi);
1118     return s;
1119 }
1120
1121 always_inline uword
1122 lookup_mpls_dst (vlib_main_t * vm,
1123                 vlib_node_runtime_t * node,
1124                 vlib_frame_t * from_frame)
1125 {
1126     return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
1127 }
1128
1129 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
1130     .function = lookup_mpls_dst,
1131     .name = "lookup-mpls-dst",
1132     .vector_size = sizeof (u32),
1133     .sibling_of = "mpls-lookup",
1134     .format_trace = format_lookup_mpls_trace,
1135     .n_next_nodes = 0,
1136 };
1137 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_node, lookup_mpls_dst)
1138
1139 always_inline uword
1140 lookup_mpls_dst_itf (vlib_main_t * vm,
1141                     vlib_node_runtime_t * node,
1142                     vlib_frame_t * from_frame)
1143 {
1144     return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
1145 }
1146
1147 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
1148     .function = lookup_mpls_dst_itf,
1149     .name = "lookup-mpls-dst-itf",
1150     .vector_size = sizeof (u32),
1151     .sibling_of = "mpls-lookup",
1152     .format_trace = format_lookup_mpls_trace,
1153     .n_next_nodes = 0,
1154 };
1155 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf)
1156
1157 typedef enum lookup_ip_dst_mcast_next_t_ {
1158     LOOKUP_IP_DST_MCAST_NEXT_RPF,
1159     LOOKUP_IP_DST_MCAST_N_NEXT,
1160 } mfib_forward_lookup_next_t;
1161
1162 always_inline uword
1163 lookup_dpo_ip_dst_mcast_inline (vlib_main_t * vm,
1164                                 vlib_node_runtime_t * node,
1165                                 vlib_frame_t * from_frame,
1166                                 int is_v4)
1167 {
1168     u32 n_left_from, next_index, * from, * to_next;
1169
1170     from = vlib_frame_vector_args (from_frame);
1171     n_left_from = from_frame->n_vectors;
1172
1173     next_index = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1174
1175     while (n_left_from > 0)
1176     {
1177         u32 n_left_to_next;
1178
1179         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
1180
1181         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1182         /*   } */
1183
1184         while (n_left_from > 0 && n_left_to_next > 0)
1185         {
1186             u32 bi0, lkdi0, fib_index0,  next0;
1187             const lookup_dpo_t * lkd0;
1188             fib_node_index_t mfei0;
1189             vlib_buffer_t * b0;
1190
1191             bi0 = from[0];
1192             to_next[0] = bi0;
1193             from += 1;
1194             to_next += 1;
1195             n_left_from -= 1;
1196             n_left_to_next -= 1;
1197
1198             b0 = vlib_get_buffer (vm, bi0);
1199
1200             /* dst lookup was done by mpls lookup */
1201             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1202             lkd0 = lookup_dpo_get(lkdi0);
1203             fib_index0 = lkd0->lkd_fib_index;
1204             next0 = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1205
1206             if (is_v4)
1207             {
1208                 ip4_header_t * ip0;
1209
1210                 ip0 = vlib_buffer_get_current (b0);
1211                 mfei0 = ip4_mfib_table_lookup(ip4_mfib_get(fib_index0),
1212                                               &ip0->src_address,
1213                                               &ip0->dst_address,
1214                                               64);
1215                 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1216                 {
1217                     lookup_trace_t *tr = vlib_add_trace (vm, node,
1218                                                          b0, sizeof (*tr));
1219                     tr->fib_index = fib_index0;
1220                     tr->lbi = mfei0;
1221                     tr->addr.ip4 = ip0->dst_address;
1222                 }
1223             }
1224             else
1225             {
1226                 ip6_header_t * ip0;
1227
1228                 ip0 = vlib_buffer_get_current (b0);
1229                 mfei0 = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index0),
1230                                                &ip0->src_address,
1231                                                &ip0->dst_address);
1232                 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1233                 {
1234                     lookup_trace_t *tr = vlib_add_trace (vm, node,
1235                                                          b0, sizeof (*tr));
1236                     tr->fib_index = fib_index0;
1237                     tr->lbi = mfei0;
1238                     tr->addr.ip6 = ip0->dst_address;
1239                 }
1240             }
1241
1242             vnet_buffer (b0)->ip.adj_index[VLIB_TX] = mfei0;
1243
1244             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1245                                             n_left_to_next, bi0, next0);
1246         }
1247         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1248     }
1249     return from_frame->n_vectors;
1250 }
1251
1252 always_inline uword
1253 lookup_ip4_dst_mcast (vlib_main_t * vm,
1254                       vlib_node_runtime_t * node,
1255                       vlib_frame_t * from_frame)
1256 {
1257     return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 1));
1258 }
1259
1260 VLIB_REGISTER_NODE (lookup_ip4_dst_mcast_node) = {
1261     .function = lookup_ip4_dst_mcast,
1262     .name = "lookup-ip4-dst-mcast",
1263     .vector_size = sizeof (u32),
1264
1265     .format_trace = format_lookup_trace,
1266     .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1267     .next_nodes = {
1268         [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip4-mfib-forward-rpf",
1269     },
1270 };
1271 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_mcast_node,
1272                               lookup_ip4_dst_mcast)
1273
1274 always_inline uword
1275 lookup_ip6_dst_mcast (vlib_main_t * vm,
1276                       vlib_node_runtime_t * node,
1277                       vlib_frame_t * from_frame)
1278 {
1279     return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 0));
1280 }
1281
1282 VLIB_REGISTER_NODE (lookup_ip6_dst_mcast_node) = {
1283     .function = lookup_ip6_dst_mcast,
1284     .name = "lookup-ip6-dst-mcast",
1285     .vector_size = sizeof (u32),
1286
1287     .format_trace = format_lookup_trace,
1288     .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1289     .next_nodes = {
1290         [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip6-mfib-forward-rpf",
1291     },
1292 };
1293 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_mcast_node,
1294                               lookup_ip6_dst_mcast)
1295
1296 static void
1297 lookup_dpo_mem_show (void)
1298 {
1299     fib_show_memory_usage("Lookup",
1300                           pool_elts(lookup_dpo_pool),
1301                           pool_len(lookup_dpo_pool),
1302                           sizeof(lookup_dpo_t));
1303 }
1304
1305 const static dpo_vft_t lkd_vft = {
1306     .dv_lock = lookup_dpo_lock,
1307     .dv_unlock = lookup_dpo_unlock,
1308     .dv_format = format_lookup_dpo,
1309 };
1310 const static dpo_vft_t lkd_vft_w_mem_show = {
1311     .dv_lock = lookup_dpo_lock,
1312     .dv_unlock = lookup_dpo_unlock,
1313     .dv_format = format_lookup_dpo,
1314     .dv_mem_show = lookup_dpo_mem_show,
1315 };
1316
1317 const static char* const lookup_src_ip4_nodes[] =
1318 {
1319     "lookup-ip4-src",
1320     NULL,
1321 };
1322 const static char* const lookup_src_ip6_nodes[] =
1323 {
1324     "lookup-ip6-src",
1325     NULL,
1326 };
1327 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
1328 {
1329     [DPO_PROTO_IP4]  = lookup_src_ip4_nodes,
1330     [DPO_PROTO_IP6]  = lookup_src_ip6_nodes,
1331     [DPO_PROTO_MPLS] = NULL,
1332 };
1333
1334 const static char* const lookup_dst_ip4_nodes[] =
1335 {
1336     "lookup-ip4-dst",
1337     NULL,
1338 };
1339 const static char* const lookup_dst_ip6_nodes[] =
1340 {
1341     "lookup-ip6-dst",
1342     NULL,
1343 };
1344 const static char* const lookup_dst_mpls_nodes[] =
1345 {
1346     "lookup-mpls-dst",
1347     NULL,
1348 };
1349 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
1350 {
1351     [DPO_PROTO_IP4]  = lookup_dst_ip4_nodes,
1352     [DPO_PROTO_IP6]  = lookup_dst_ip6_nodes,
1353     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
1354 };
1355
1356 const static char* const lookup_dst_mcast_ip4_nodes[] =
1357 {
1358     "lookup-ip4-dst-mcast",
1359     NULL,
1360 };
1361 const static char* const lookup_dst_mcast_ip6_nodes[] =
1362 {
1363     "lookup-ip6-dst-mcast",
1364     NULL,
1365 };
1366 const static char* const * const lookup_dst_mcast_nodes[DPO_PROTO_NUM] =
1367 {
1368     [DPO_PROTO_IP4]  = lookup_dst_mcast_ip4_nodes,
1369     [DPO_PROTO_IP6]  = lookup_dst_mcast_ip6_nodes,
1370 };
1371
1372 const static char* const lookup_dst_from_interface_ip4_nodes[] =
1373 {
1374     "lookup-ip4-dst-itf",
1375     NULL,
1376 };
1377 const static char* const lookup_dst_from_interface_ip6_nodes[] =
1378 {
1379     "lookup-ip6-dst-itf",
1380     NULL,
1381 };
1382 const static char* const lookup_dst_from_interface_mpls_nodes[] =
1383 {
1384     "lookup-mpls-dst-itf",
1385     NULL,
1386 };
1387 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
1388 {
1389     [DPO_PROTO_IP4]  = lookup_dst_from_interface_ip4_nodes,
1390     [DPO_PROTO_IP6]  = lookup_dst_from_interface_ip6_nodes,
1391     [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
1392 };
1393
1394
1395 void
1396 lookup_dpo_module_init (void)
1397 {
1398     dpo_register(DPO_LOOKUP, &lkd_vft_w_mem_show, NULL);
1399
1400     /*
1401      * There are various sorts of lookup; src or dst addr v4 /v6 etc.
1402      * there isn't an object type for each (there is only the lookup_dpo_t),
1403      * but, for performance reasons, there is a data plane function, and hence
1404      * VLIB node for each. VLIB graph node construction is based on DPO types
1405      * so we create sub-types.
1406      */
1407     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
1408         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
1409     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
1410         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
1411     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST] =
1412         dpo_register_new_type(&lkd_vft, lookup_dst_mcast_nodes);
1413     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
1414         dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes);
1415 }