src/vnet/dpo/lookup_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/lookup_dpo.h>
  18 #include <vnet/dpo/load_balance_map.h>
  19 #include <vnet/mpls/mpls_lookup.h>
  20 #include <vnet/fib/fib_table.h>
  21 #include <vnet/fib/ip4_fib.h>
  22 #include <vnet/fib/ip6_fib.h>
  23 #include <vnet/fib/mpls_fib.h>
  24 #include <vnet/mfib/mfib_table.h>
  25 #include <vnet/mfib/ip4_mfib.h>
  26 #include <vnet/mfib/ip6_mfib.h>
  27
  28 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
  29 static const char *const lookup_cast_names[] = LOOKUP_CASTS;
  30
  31 /**
  32  * If a packet encounters a lookup DPO more than the many times
  33  * then we assume there is a loop in the forward graph and drop the packet
  34  */
  35 #define MAX_LUKPS_PER_PACKET 4
  36
  37 /**
  38  * @brief Enumeration of the lookup subtypes
  39  */
  40 typedef enum lookup_sub_type_t_
  41 {
  42     LOOKUP_SUB_TYPE_SRC,
  43     LOOKUP_SUB_TYPE_DST,
  44     LOOKUP_SUB_TYPE_DST_MCAST,
  45     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
  46 } lookup_sub_type_t;
  47 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
  48
  49 #define FOR_EACH_LOOKUP_SUB_TYPE(_st)                                   \
  50     for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
  51
  52 /**
  53  * @brief pool of all MPLS Label DPOs
  54  */
  55 lookup_dpo_t *lookup_dpo_pool;
  56
  57 /**
  58  * @brief An array of registered DPO type values for the sub-types
  59  */
  60 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
  61
  62 static lookup_dpo_t *
  63 lookup_dpo_alloc (void)
  64 {
  65     lookup_dpo_t *lkd;
  66
  67     pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
  68
  69     return (lkd);
  70 }
  71
  72 static index_t
  73 lookup_dpo_get_index (lookup_dpo_t *lkd)
  74 {
  75     return (lkd - lookup_dpo_pool);
  76 }
  77
  78 static void
  79 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
  80                           dpo_proto_t proto,
  81                           lookup_cast_t cast,
  82                           lookup_input_t input,
  83                           lookup_table_t table_config,
  84                           dpo_id_t *dpo)
  85 {
  86     lookup_dpo_t *lkd;
  87     dpo_type_t type;
  88
  89     lkd = lookup_dpo_alloc();
  90     lkd->lkd_fib_index = fib_index;
  91     lkd->lkd_proto = proto;
  92     lkd->lkd_input = input;
  93     lkd->lkd_table = table_config;
  94     lkd->lkd_cast  = cast;
  95
  96     /*
  97      * use the input type to select the lookup sub-type
  98      */
  99     type = 0;
 100
 101     switch (input)
 102     {
 103     case LOOKUP_INPUT_SRC_ADDR:
 104         type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
 105         break;
 106     case LOOKUP_INPUT_DST_ADDR:
 107         switch (table_config)
 108         {
 109         case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
 110             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
 111             break;
 112         case LOOKUP_TABLE_FROM_CONFIG:
 113             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
 114             break;
 115         }
 116         if (LOOKUP_MULTICAST == cast)
 117         {
 118             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST];
 119         }
 120     }
 121
 122     if (0 == type)
 123     {
 124         dpo_reset(dpo);
 125     }
 126     else
 127     {
 128         dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
 129     }
 130 }
 131
 132 void
 133 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
 134                                     dpo_proto_t proto,
 135                                     lookup_cast_t cast,
 136                                     lookup_input_t input,
 137                                     lookup_table_t table_config,
 138                                     dpo_id_t *dpo)
 139 {
 140     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 141     {
 142         if (LOOKUP_UNICAST == cast)
 143         {
 144             fib_table_lock(fib_index,
 145                            dpo_proto_to_fib(proto),
 146                            FIB_SOURCE_RR);
 147         }
 148         else
 149         {
 150             mfib_table_lock(fib_index,
 151                             dpo_proto_to_fib(proto),
 152                             MFIB_SOURCE_RR);
 153         }
 154     }
 155     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 156 }
 157
 158 void
 159 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 160                                    dpo_proto_t proto,
 161                                    lookup_cast_t cast,
 162                                    lookup_input_t input,
 163                                    lookup_table_t table_config,
 164                                    dpo_id_t *dpo)
 165 {
 166     fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
 167
 168     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 169     {
 170         if (LOOKUP_UNICAST == cast)
 171         {
 172             fib_index =
 173                 fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 174                                                   table_id,
 175                                                   FIB_SOURCE_RR);
 176         }
 177         else
 178         {
 179             fib_index =
 180                 mfib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 181                                                    table_id,
 182                                                    MFIB_SOURCE_RR);
 183         }
 184     }
 185
 186     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
 187     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 188 }
 189
 190 u8*
 191 format_lookup_dpo (u8 *s, va_list *args)
 192 {
 193     index_t index = va_arg (*args, index_t);
 194     lookup_dpo_t *lkd;
 195
 196     lkd = lookup_dpo_get(index);
 197
 198     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
 199     {
 200         s = format(s, "%s,%s lookup in interface's %U table",
 201                    lookup_input_names[lkd->lkd_input],
 202                    lookup_cast_names[lkd->lkd_cast],
 203                    format_dpo_proto, lkd->lkd_proto);
 204     }
 205     else
 206     {
 207         if (LOOKUP_UNICAST == lkd->lkd_cast)
 208         {
 209             s = format(s, "%s,%s lookup in %U",
 210                        lookup_input_names[lkd->lkd_input],
 211                        lookup_cast_names[lkd->lkd_cast],
 212                        format_fib_table_name, lkd->lkd_fib_index,
 213                        dpo_proto_to_fib(lkd->lkd_proto));
 214         }
 215         else
 216         {
 217             s = format(s, "%s,%s lookup in %U",
 218                        lookup_input_names[lkd->lkd_input],
 219                        lookup_cast_names[lkd->lkd_cast],
 220                        format_mfib_table_name, lkd->lkd_fib_index,
 221                        dpo_proto_to_fib(lkd->lkd_proto));
 222         }
 223     }
 224     return (s);
 225 }
 226
 227 static void
 228 lookup_dpo_lock (dpo_id_t *dpo)
 229 {
 230     lookup_dpo_t *lkd;
 231
 232     lkd = lookup_dpo_get(dpo->dpoi_index);
 233
 234     lkd->lkd_locks++;
 235 }
 236
 237 static void
 238 lookup_dpo_unlock (dpo_id_t *dpo)
 239 {
 240     lookup_dpo_t *lkd;
 241
 242     lkd = lookup_dpo_get(dpo->dpoi_index);
 243
 244     lkd->lkd_locks--;
 245
 246     if (0 == lkd->lkd_locks)
 247     {
 248         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
 249         {
 250             if (LOOKUP_UNICAST == lkd->lkd_cast)
 251             {
 252                 fib_table_unlock(lkd->lkd_fib_index,
 253                                  dpo_proto_to_fib(lkd->lkd_proto),
 254                                  FIB_SOURCE_RR);
 255             }
 256             else
 257             {
 258                 mfib_table_unlock(lkd->lkd_fib_index,
 259                                   dpo_proto_to_fib(lkd->lkd_proto),
 260                                   MFIB_SOURCE_RR);
 261             }
 262         }
 263         pool_put(lookup_dpo_pool, lkd);
 264     }
 265 }
 266
 267 always_inline void
 268 ip4_src_fib_lookup_one (u32 src_fib_index0,
 269                         const ip4_address_t * addr0,
 270                         u32 * src_adj_index0)
 271 {
 272     ip4_fib_mtrie_leaf_t leaf0;
 273     ip4_fib_mtrie_t * mtrie0;
 274
 275     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 276
 277     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 278     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 279     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 280
 281     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 282 }
 283
 284 always_inline void
 285 ip4_src_fib_lookup_two (u32 src_fib_index0,
 286                         u32 src_fib_index1,
 287                         const ip4_address_t * addr0,
 288                         const ip4_address_t * addr1,
 289                         u32 * src_adj_index0,
 290                         u32 * src_adj_index1)
 291 {
 292     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 293     ip4_fib_mtrie_t * mtrie0, * mtrie1;
 294
 295     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 296     mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
 297
 298     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 299     leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, addr1);
 300
 301     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 302     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
 303
 304     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 305     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
 306
 307     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 308     src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 309 }
 310
 311 /**
 312  * @brief Lookup trace  data
 313  */
 314 typedef struct lookup_trace_t_
 315 {
 316     union {
 317         ip46_address_t addr;
 318         mpls_unicast_header_t hdr;
 319     };
 320     fib_node_index_t fib_index;
 321     index_t lbi;
 322 } lookup_trace_t;
 323
 324
 325 always_inline uword
 326 lookup_dpo_ip4_inline (vlib_main_t * vm,
 327                        vlib_node_runtime_t * node,
 328                        vlib_frame_t * from_frame,
 329                        int input_src_addr,
 330                        int table_from_interface)
 331 {
 332     u32 n_left_from, next_index, * from, * to_next;
 333     u32 thread_index = vlib_get_thread_index();
 334     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 335
 336     from = vlib_frame_vector_args (from_frame);
 337     n_left_from = from_frame->n_vectors;
 338
 339     next_index = node->cached_next_index;
 340
 341     while (n_left_from > 0)
 342     {
 343         u32 n_left_to_next;
 344
 345         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 346
 347         while (n_left_from >= 4 && n_left_to_next > 2)
 348         {
 349             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 350             flow_hash_config_t flow_hash_config0;
 351             const ip4_address_t *input_addr0;
 352             const load_balance_t *lb0;
 353             const lookup_dpo_t * lkd0;
 354             const ip4_header_t * ip0;
 355             const dpo_id_t *dpo0;
 356             vlib_buffer_t * b0;
 357             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 358             flow_hash_config_t flow_hash_config1;
 359             const ip4_address_t *input_addr1;
 360             const load_balance_t *lb1;
 361             const lookup_dpo_t * lkd1;
 362             const ip4_header_t * ip1;
 363             const dpo_id_t *dpo1;
 364             vlib_buffer_t * b1;
 365
 366             /* Prefetch next iteration. */
 367             {
 368                 vlib_buffer_t * p2, * p3;
 369
 370                 p2 = vlib_get_buffer (vm, from[2]);
 371                 p3 = vlib_get_buffer (vm, from[3]);
 372
 373                 vlib_prefetch_buffer_header (p2, LOAD);
 374                 vlib_prefetch_buffer_header (p3, LOAD);
 375
 376                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 377                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 378             }
 379
 380             bi0 = from[0];
 381             to_next[0] = bi0;
 382             bi1 = from[1];
 383             to_next[1] = bi1;
 384             from += 2;
 385             to_next += 2;
 386             n_left_from -= 2;
 387             n_left_to_next -= 2;
 388
 389             b0 = vlib_get_buffer (vm, bi0);
 390             ip0 = vlib_buffer_get_current (b0);
 391             b1 = vlib_get_buffer (vm, bi1);
 392             ip1 = vlib_buffer_get_current (b1);
 393
 394             /* dst lookup was done by ip4 lookup */
 395             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 396             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 397             lkd0 = lookup_dpo_get(lkdi0);
 398             lkd1 = lookup_dpo_get(lkdi1);
 399
 400             /*
 401              * choose between a lookup using the fib index in the DPO
 402              * or getting the FIB index from the interface.
 403              */
 404             if (table_from_interface)
 405             {
 406                 fib_index0 =
 407                     ip4_fib_table_get_index_for_sw_if_index(
 408                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 409                 fib_index1 =
 410                     ip4_fib_table_get_index_for_sw_if_index(
 411                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 412             }
 413             else
 414             {
 415                 fib_index0 = lkd0->lkd_fib_index;
 416                 fib_index1 = lkd1->lkd_fib_index;
 417             }
 418
 419             /*
 420              * choose between a source or destination address lookup in the table
 421              */
 422             if (input_src_addr)
 423             {
 424                 input_addr0 = &ip0->src_address;
 425                 input_addr1 = &ip1->src_address;
 426             }
 427             else
 428             {
 429                 input_addr0 = &ip0->dst_address;
 430                 input_addr1 = &ip1->dst_address;
 431             }
 432
 433             /* do lookup */
 434             ip4_src_fib_lookup_two (fib_index0, fib_index1,
 435                                     input_addr0, input_addr1,
 436                                     &lbi0, &lbi1);
 437             lb0 = load_balance_get(lbi0);
 438             lb1 = load_balance_get(lbi1);
 439
 440             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 441             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 442
 443             /* Use flow hash to compute multipath adjacency. */
 444             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 445             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 446
 447             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 448             {
 449                 flow_hash_config0 = lb0->lb_hash_config;
 450                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 451                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 452             }
 453
 454             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 455             {
 456                 flow_hash_config1 = lb1->lb_hash_config;
 457                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 458                     ip4_compute_flow_hash (ip1, flow_hash_config1);
 459             }
 460
 461             dpo0 = load_balance_get_bucket_i(lb0,
 462                                              (hash_c0 &
 463                                               (lb0->lb_n_buckets_minus_1)));
 464             dpo1 = load_balance_get_bucket_i(lb1,
 465                                              (hash_c1 &
 466                                               (lb1->lb_n_buckets_minus_1)));
 467
 468             next0 = dpo0->dpoi_next_node;
 469             next1 = dpo1->dpoi_next_node;
 470             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 471             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 472
 473             vlib_increment_combined_counter
 474                 (cm, thread_index, lbi0, 1,
 475                  vlib_buffer_length_in_chain (vm, b0));
 476             vlib_increment_combined_counter
 477                 (cm, thread_index, lbi1, 1,
 478                  vlib_buffer_length_in_chain (vm, b1));
 479
 480             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 481                 vnet_buffer2(b0)->loop_counter = 0;
 482                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 483             }
 484             if (!(b1->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 485                 vnet_buffer2(b1)->loop_counter = 0;
 486                 b1->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 487             }
 488
 489             vnet_buffer2(b0)->loop_counter++;
 490             vnet_buffer2(b1)->loop_counter++;
 491
 492             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 493                 next0 = IP_LOOKUP_NEXT_DROP;
 494             if (PREDICT_FALSE(vnet_buffer2(b1)->loop_counter > MAX_LUKPS_PER_PACKET))
 495                 next1 = IP_LOOKUP_NEXT_DROP;
 496
 497             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 498             {
 499                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 500                                                      b0, sizeof (*tr));
 501                 tr->fib_index = fib_index0;
 502                 tr->lbi = lbi0;
 503                 tr->addr.ip4 = *input_addr0;
 504             }
 505             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 506             {
 507                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 508                                                      b1, sizeof (*tr));
 509                 tr->fib_index = fib_index1;
 510                 tr->lbi = lbi1;
 511                 tr->addr.ip4 = *input_addr1;
 512             }
 513
 514             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 515                                              to_next, n_left_to_next,
 516                                              bi0, bi1, next0, next1);
 517         }
 518
 519         while (n_left_from > 0 && n_left_to_next > 0)
 520         {
 521             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 522             flow_hash_config_t flow_hash_config0;
 523             const ip4_address_t *input_addr;
 524             const load_balance_t *lb0;
 525             const lookup_dpo_t * lkd0;
 526             const ip4_header_t * ip0;
 527             const dpo_id_t *dpo0;
 528             vlib_buffer_t * b0;
 529
 530             bi0 = from[0];
 531             to_next[0] = bi0;
 532             from += 1;
 533             to_next += 1;
 534             n_left_from -= 1;
 535             n_left_to_next -= 1;
 536
 537             b0 = vlib_get_buffer (vm, bi0);
 538             ip0 = vlib_buffer_get_current (b0);
 539
 540             /* dst lookup was done by ip4 lookup */
 541             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 542             lkd0 = lookup_dpo_get(lkdi0);
 543
 544             /*
 545              * choose between a lookup using the fib index in the DPO
 546              * or getting the FIB index from the interface.
 547              */
 548             if (table_from_interface)
 549             {
 550                 fib_index0 =
 551                     ip4_fib_table_get_index_for_sw_if_index(
 552                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 553             }
 554             else
 555             {
 556                 fib_index0 = lkd0->lkd_fib_index;
 557             }
 558
 559             /*
 560              * choose between a source or destination address lookup in the table
 561              */
 562             if (input_src_addr)
 563             {
 564                 input_addr = &ip0->src_address;
 565             }
 566             else
 567             {
 568                 input_addr = &ip0->dst_address;
 569             }
 570
 571             /* do lookup */
 572             ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
 573             lb0 = load_balance_get(lbi0);
 574
 575             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 576
 577             /* Use flow hash to compute multipath adjacency. */
 578             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 579
 580             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 581             {
 582                 flow_hash_config0 = lb0->lb_hash_config;
 583                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 584                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 585             }
 586
 587             dpo0 = load_balance_get_bucket_i(lb0,
 588                                              (hash_c0 &
 589                                               (lb0->lb_n_buckets_minus_1)));
 590
 591             next0 = dpo0->dpoi_next_node;
 592             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 593
 594             vlib_increment_combined_counter
 595                 (cm, thread_index, lbi0, 1,
 596                  vlib_buffer_length_in_chain (vm, b0));
 597
 598             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 599                 vnet_buffer2(b0)->loop_counter = 0;
 600                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 601             }
 602
 603             vnet_buffer2(b0)->loop_counter++;
 604
 605             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 606                 next0 = IP_LOOKUP_NEXT_DROP;
 607
 608             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 609             {
 610                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 611                                                      b0, sizeof (*tr));
 612                 tr->fib_index = fib_index0;
 613                 tr->lbi = lbi0;
 614                 tr->addr.ip4 = *input_addr;
 615             }
 616
 617             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 618                                             n_left_to_next, bi0, next0);
 619         }
 620         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 621     }
 622     return from_frame->n_vectors;
 623 }
 624
 625 static u8 *
 626 format_lookup_trace (u8 * s, va_list * args)
 627 {
 628     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 629     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 630     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 631     u32 indent = format_get_indent (s);
 632     s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
 633                 format_white_space, indent,
 634                 t->fib_index,
 635                 format_ip46_address, &t->addr, IP46_TYPE_ANY,
 636                 t->lbi);
 637     return s;
 638 }
 639
 640 static uword
 641 lookup_ip4_dst (vlib_main_t * vm,
 642                 vlib_node_runtime_t * node,
 643                 vlib_frame_t * from_frame)
 644 {
 645     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
 646 }
 647
 648 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
 649     .function = lookup_ip4_dst,
 650     .name = "lookup-ip4-dst",
 651     .vector_size = sizeof (u32),
 652     .sibling_of = "ip4-lookup",
 653     .format_trace = format_lookup_trace,
 654 };
 655 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_node, lookup_ip4_dst)
 656
 657 static uword
 658 lookup_ip4_dst_itf (vlib_main_t * vm,
 659                     vlib_node_runtime_t * node,
 660                     vlib_frame_t * from_frame)
 661 {
 662     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
 663 }
 664
 665 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
 666     .function = lookup_ip4_dst_itf,
 667     .name = "lookup-ip4-dst-itf",
 668     .vector_size = sizeof (u32),
 669     .sibling_of = "ip4-lookup",
 670     .format_trace = format_lookup_trace,
 671 };
 672 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_itf_node, lookup_ip4_dst_itf)
 673
 674 static uword
 675 lookup_ip4_src (vlib_main_t * vm,
 676                 vlib_node_runtime_t * node,
 677                 vlib_frame_t * from_frame)
 678 {
 679     return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
 680 }
 681
 682 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
 683     .function = lookup_ip4_src,
 684     .name = "lookup-ip4-src",
 685     .vector_size = sizeof (u32),
 686     .format_trace = format_lookup_trace,
 687     .sibling_of = "ip4-lookup",
 688 };
 689 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_src_node, lookup_ip4_src)
 690
 691 always_inline uword
 692 lookup_dpo_ip6_inline (vlib_main_t * vm,
 693                        vlib_node_runtime_t * node,
 694                        vlib_frame_t * from_frame,
 695                        int input_src_addr,
 696                        int table_from_interface)
 697 {
 698     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 699     u32 n_left_from, next_index, * from, * to_next;
 700     u32 thread_index = vlib_get_thread_index();
 701
 702     from = vlib_frame_vector_args (from_frame);
 703     n_left_from = from_frame->n_vectors;
 704
 705     next_index = node->cached_next_index;
 706
 707     while (n_left_from > 0)
 708     {
 709         u32 n_left_to_next;
 710
 711         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 712
 713         while (n_left_from >= 4 && n_left_to_next > 2)
 714         {
 715             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 716             flow_hash_config_t flow_hash_config0;
 717             const ip6_address_t *input_addr0;
 718             const load_balance_t *lb0;
 719             const lookup_dpo_t * lkd0;
 720             const ip6_header_t * ip0;
 721             const dpo_id_t *dpo0;
 722             vlib_buffer_t * b0;
 723             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 724             flow_hash_config_t flow_hash_config1;
 725             const ip6_address_t *input_addr1;
 726             const load_balance_t *lb1;
 727             const lookup_dpo_t * lkd1;
 728             const ip6_header_t * ip1;
 729             const dpo_id_t *dpo1;
 730             vlib_buffer_t * b1;
 731
 732             /* Prefetch next iteration. */
 733             {
 734                 vlib_buffer_t * p2, * p3;
 735
 736                 p2 = vlib_get_buffer (vm, from[2]);
 737                 p3 = vlib_get_buffer (vm, from[3]);
 738
 739                 vlib_prefetch_buffer_header (p2, LOAD);
 740                 vlib_prefetch_buffer_header (p3, LOAD);
 741
 742                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 743                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 744             }
 745
 746             bi0 = from[0];
 747             to_next[0] = bi0;
 748             bi1 = from[1];
 749             to_next[1] = bi1;
 750             from += 2;
 751             to_next += 2;
 752             n_left_from -= 2;
 753             n_left_to_next -= 2;
 754
 755             b0 = vlib_get_buffer (vm, bi0);
 756             ip0 = vlib_buffer_get_current (b0);
 757             b1 = vlib_get_buffer (vm, bi1);
 758             ip1 = vlib_buffer_get_current (b1);
 759
 760             /* dst lookup was done by ip6 lookup */
 761             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 762             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 763             lkd0 = lookup_dpo_get(lkdi0);
 764             lkd1 = lookup_dpo_get(lkdi1);
 765
 766             /*
 767              * choose between a lookup using the fib index in the DPO
 768              * or getting the FIB index from the interface.
 769              */
 770             if (table_from_interface)
 771             {
 772                 fib_index0 =
 773                     ip6_fib_table_get_index_for_sw_if_index(
 774                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 775                 fib_index1 =
 776                     ip6_fib_table_get_index_for_sw_if_index(
 777                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 778             }
 779             else
 780             {
 781                 fib_index0 = lkd0->lkd_fib_index;
 782                 fib_index1 = lkd1->lkd_fib_index;
 783             }
 784
 785             /*
 786              * choose between a source or destination address lookup in the table
 787              */
 788             if (input_src_addr)
 789             {
 790                 input_addr0 = &ip0->src_address;
 791                 input_addr1 = &ip1->src_address;
 792             }
 793             else
 794             {
 795                 input_addr0 = &ip0->dst_address;
 796                 input_addr1 = &ip1->dst_address;
 797             }
 798
 799             /* do src lookup */
 800             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 801                                                fib_index0,
 802                                                input_addr0);
 803             lbi1 = ip6_fib_table_fwding_lookup(&ip6_main,
 804                                                fib_index1,
 805                                                input_addr1);
 806             lb0 = load_balance_get(lbi0);
 807             lb1 = load_balance_get(lbi1);
 808
 809             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 810             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 811
 812             /* Use flow hash to compute multipath adjacency. */
 813             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 814             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 815
 816             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 817                 vnet_buffer2(b0)->loop_counter = 0;
 818                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 819             }
 820             if (!(b1->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 821                 vnet_buffer2(b1)->loop_counter = 0;
 822                 b1->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 823             }
 824
 825             vnet_buffer2(b0)->loop_counter++;
 826             vnet_buffer2(b1)->loop_counter++;
 827
 828             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 829                 next0 = IP_LOOKUP_NEXT_DROP;
 830             if (PREDICT_FALSE(vnet_buffer2(b1)->loop_counter > MAX_LUKPS_PER_PACKET))
 831                 next1 = IP_LOOKUP_NEXT_DROP;
 832
 833             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 834             {
 835                 flow_hash_config0 = lb0->lb_hash_config;
 836                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 837                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 838             }
 839
 840             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 841             {
 842                 flow_hash_config1 = lb1->lb_hash_config;
 843                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 844                     ip6_compute_flow_hash (ip1, flow_hash_config1);
 845             }
 846
 847             dpo0 = load_balance_get_bucket_i(lb0,
 848                                              (hash_c0 &
 849                                               (lb0->lb_n_buckets_minus_1)));
 850             dpo1 = load_balance_get_bucket_i(lb1,
 851                                              (hash_c1 &
 852                                               (lb1->lb_n_buckets_minus_1)));
 853
 854             next0 = dpo0->dpoi_next_node;
 855             next1 = dpo1->dpoi_next_node;
 856             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 857             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 858
 859             vlib_increment_combined_counter
 860                 (cm, thread_index, lbi0, 1,
 861                  vlib_buffer_length_in_chain (vm, b0));
 862             vlib_increment_combined_counter
 863                 (cm, thread_index, lbi1, 1,
 864                  vlib_buffer_length_in_chain (vm, b1));
 865
 866             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 867             {
 868                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 869                                                      b0, sizeof (*tr));
 870                 tr->fib_index = fib_index0;
 871                 tr->lbi = lbi0;
 872                 tr->addr.ip6 = *input_addr0;
 873             }
 874             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 875             {
 876                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 877                                                      b1, sizeof (*tr));
 878                 tr->fib_index = fib_index1;
 879                 tr->lbi = lbi1;
 880                 tr->addr.ip6 = *input_addr1;
 881             }
 882             vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
 883                                             n_left_to_next, bi0, bi1,
 884                                             next0, next1);
 885         }
 886         while (n_left_from > 0 && n_left_to_next > 0)
 887         {
 888             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 889             flow_hash_config_t flow_hash_config0;
 890             const ip6_address_t *input_addr0;
 891             const load_balance_t *lb0;
 892             const lookup_dpo_t * lkd0;
 893             const ip6_header_t * ip0;
 894             const dpo_id_t *dpo0;
 895             vlib_buffer_t * b0;
 896
 897             bi0 = from[0];
 898             to_next[0] = bi0;
 899             from += 1;
 900             to_next += 1;
 901             n_left_from -= 1;
 902             n_left_to_next -= 1;
 903
 904             b0 = vlib_get_buffer (vm, bi0);
 905             ip0 = vlib_buffer_get_current (b0);
 906
 907             /* dst lookup was done by ip6 lookup */
 908             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 909             lkd0 = lookup_dpo_get(lkdi0);
 910
 911             /*
 912              * choose between a lookup using the fib index in the DPO
 913              * or getting the FIB index from the interface.
 914              */
 915             if (table_from_interface)
 916             {
 917                 fib_index0 =
 918                     ip6_fib_table_get_index_for_sw_if_index(
 919                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 920             }
 921             else
 922             {
 923                 fib_index0 = lkd0->lkd_fib_index;
 924             }
 925
 926             /*
 927              * choose between a source or destination address lookup in the table
 928              */
 929             if (input_src_addr)
 930             {
 931                 input_addr0 = &ip0->src_address;
 932             }
 933             else
 934             {
 935                 input_addr0 = &ip0->dst_address;
 936             }
 937
 938             /* do src lookup */
 939             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 940                                                fib_index0,
 941                                                input_addr0);
 942             lb0 = load_balance_get(lbi0);
 943
 944             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 945
 946             /* Use flow hash to compute multipath adjacency. */
 947             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 948
 949             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 950             {
 951                 flow_hash_config0 = lb0->lb_hash_config;
 952                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 953                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 954             }
 955
 956             dpo0 = load_balance_get_bucket_i(lb0,
 957                                              (hash_c0 &
 958                                               (lb0->lb_n_buckets_minus_1)));
 959
 960             next0 = dpo0->dpoi_next_node;
 961             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 962
 963             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 964                 vnet_buffer2(b0)->loop_counter = 0;
 965                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 966             }
 967
 968             vnet_buffer2(b0)->loop_counter++;
 969
 970             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 971                 next0 = IP_LOOKUP_NEXT_DROP;
 972
 973             vlib_increment_combined_counter
 974                 (cm, thread_index, lbi0, 1,
 975                  vlib_buffer_length_in_chain (vm, b0));
 976
 977             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 978             {
 979                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 980                                                      b0, sizeof (*tr));
 981                 tr->fib_index = fib_index0;
 982                 tr->lbi = lbi0;
 983                 tr->addr.ip6 = *input_addr0;
 984             }
 985             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 986                                             n_left_to_next, bi0, next0);
 987         }
 988         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 989     }
 990     return from_frame->n_vectors;
 991 }
 992
 993 static uword
 994 lookup_ip6_dst (vlib_main_t * vm,
 995                 vlib_node_runtime_t * node,
 996                 vlib_frame_t * from_frame)
 997 {
 998     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 0));
 999 }
1000
1001 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
1002     .function = lookup_ip6_dst,
1003     .name = "lookup-ip6-dst",
1004     .vector_size = sizeof (u32),
1005     .format_trace = format_lookup_trace,
1006     .sibling_of = "ip6-lookup",
1007 };
1008 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_node, lookup_ip6_dst)
1009
1010 static uword
1011 lookup_ip6_dst_itf (vlib_main_t * vm,
1012                     vlib_node_runtime_t * node,
1013                     vlib_frame_t * from_frame)
1014 {
1015     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 1));
1016 }
1017
1018 VLIB_REGISTER_NODE (lookup_ip6_dst_itf_node) = {
1019     .function = lookup_ip6_dst_itf,
1020     .name = "lookup-ip6-dst-itf",
1021     .vector_size = sizeof (u32),
1022     .format_trace = format_lookup_trace,
1023     .sibling_of = "ip6-lookup",
1024 };
1025 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_itf_node, lookup_ip6_dst_itf)
1026
1027 static uword
1028 lookup_ip6_src (vlib_main_t * vm,
1029                 vlib_node_runtime_t * node,
1030                 vlib_frame_t * from_frame)
1031 {
1032     return (lookup_dpo_ip6_inline(vm, node, from_frame, 1, 0));
1033 }
1034
1035 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
1036     .function = lookup_ip6_src,
1037     .name = "lookup-ip6-src",
1038     .vector_size = sizeof (u32),
1039     .format_trace = format_lookup_trace,
1040     .sibling_of = "ip6-lookup",
1041 };
1042 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_src_node, lookup_ip6_src)
1043
1044 always_inline uword
1045 lookup_dpo_mpls_inline (vlib_main_t * vm,
1046                        vlib_node_runtime_t * node,
1047                        vlib_frame_t * from_frame,
1048                        int table_from_interface)
1049 {
1050     u32 n_left_from, next_index, * from, * to_next;
1051     u32 thread_index = vlib_get_thread_index();
1052     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
1053
1054     from = vlib_frame_vector_args (from_frame);
1055     n_left_from = from_frame->n_vectors;
1056
1057     next_index = node->cached_next_index;
1058
1059     while (n_left_from > 0)
1060     {
1061         u32 n_left_to_next;
1062
1063         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
1064
1065         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1066         /*   } */
1067
1068         while (n_left_from > 0 && n_left_to_next > 0)
1069         {
1070             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash0;
1071             const mpls_unicast_header_t * hdr0;
1072             const load_balance_t *lb0;
1073             const lookup_dpo_t * lkd0;
1074             const dpo_id_t *dpo0;
1075             vlib_buffer_t * b0;
1076
1077             bi0 = from[0];
1078             to_next[0] = bi0;
1079             from += 1;
1080             to_next += 1;
1081             n_left_from -= 1;
1082             n_left_to_next -= 1;
1083
1084             b0 = vlib_get_buffer (vm, bi0);
1085             hdr0 = vlib_buffer_get_current (b0);
1086
1087             /* dst lookup was done by mpls lookup */
1088             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1089             lkd0 = lookup_dpo_get(lkdi0);
1090
1091             /*
1092              * choose between a lookup using the fib index in the DPO
1093              * or getting the FIB index from the interface.
1094              */
1095             if (table_from_interface)
1096             {
1097                 fib_index0 =
1098                     mpls_fib_table_get_index_for_sw_if_index(
1099                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1100             }
1101             else
1102             {
1103                 fib_index0 = lkd0->lkd_fib_index;
1104             }
1105
1106             /* do lookup */
1107             lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
1108             lb0  = load_balance_get(lbi0);
1109             dpo0 = load_balance_get_bucket_i(lb0, 0);
1110
1111             next0 = dpo0->dpoi_next_node;
1112             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1113
1114
1115             if (MPLS_IS_REPLICATE & lbi0)
1116             {
1117                 next0 = mpls_lookup_to_replicate_edge;
1118                 vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
1119                     (lbi0 & ~MPLS_IS_REPLICATE);
1120             }
1121             else
1122             {
1123                 lb0 = load_balance_get(lbi0);
1124                 ASSERT (lb0->lb_n_buckets > 0);
1125                 ASSERT (is_pow2 (lb0->lb_n_buckets));
1126
1127                 if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
1128                 {
1129                     hash0 = vnet_buffer (b0)->ip.flow_hash =
1130                         mpls_compute_flow_hash(hdr0, lb0->lb_hash_config);
1131                     dpo0 = load_balance_get_fwd_bucket
1132                         (lb0,
1133                          (hash0 & (lb0->lb_n_buckets_minus_1)));
1134                 }
1135                 else
1136                 {
1137                     dpo0 = load_balance_get_bucket_i (lb0, 0);
1138                 }
1139                 next0 = dpo0->dpoi_next_node;
1140
1141                 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1142
1143                 vlib_increment_combined_counter
1144                     (cm, thread_index, lbi0, 1,
1145                      vlib_buffer_length_in_chain (vm, b0));
1146             }
1147
1148             vnet_buffer (b0)->mpls.ttl = ((char*)hdr0)[3];
1149             vnet_buffer (b0)->mpls.exp = (((char*)hdr0)[2] & 0xe) >> 1;
1150             vnet_buffer (b0)->mpls.first = 1;
1151             vlib_buffer_advance(b0, sizeof(*hdr0));
1152
1153             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
1154                 vnet_buffer2(b0)->loop_counter = 0;
1155                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
1156             }
1157
1158             vnet_buffer2(b0)->loop_counter++;
1159
1160             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
1161                 next0 = MPLS_LOOKUP_NEXT_DROP;
1162
1163             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1164             {
1165                 lookup_trace_t *tr = vlib_add_trace (vm, node,
1166                                                      b0, sizeof (*tr));
1167                 tr->fib_index = fib_index0;
1168                 tr->lbi = lbi0;
1169                 tr->hdr = *hdr0;
1170             }
1171
1172            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1173                                             n_left_to_next, bi0, next0);
1174         }
1175         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1176     }
1177     return from_frame->n_vectors;
1178 }
1179
1180 static u8 *
1181 format_lookup_mpls_trace (u8 * s, va_list * args)
1182 {
1183     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1184     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1185     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
1186     u32 indent = format_get_indent (s);
1187     mpls_unicast_header_t hdr;
1188
1189     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
1190
1191     s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
1192                 format_white_space, indent,
1193                 t->fib_index,
1194                 format_mpls_header, hdr,
1195                 t->lbi);
1196     return s;
1197 }
1198
1199 static uword
1200 lookup_mpls_dst (vlib_main_t * vm,
1201                 vlib_node_runtime_t * node,
1202                 vlib_frame_t * from_frame)
1203 {
1204     return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
1205 }
1206
1207 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
1208     .function = lookup_mpls_dst,
1209     .name = "lookup-mpls-dst",
1210     .vector_size = sizeof (u32),
1211     .sibling_of = "mpls-lookup",
1212     .format_trace = format_lookup_mpls_trace,
1213     .n_next_nodes = 0,
1214 };
1215 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_node, lookup_mpls_dst)
1216
1217 static uword
1218 lookup_mpls_dst_itf (vlib_main_t * vm,
1219                     vlib_node_runtime_t * node,
1220                     vlib_frame_t * from_frame)
1221 {
1222     return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
1223 }
1224
1225 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
1226     .function = lookup_mpls_dst_itf,
1227     .name = "lookup-mpls-dst-itf",
1228     .vector_size = sizeof (u32),
1229     .sibling_of = "mpls-lookup",
1230     .format_trace = format_lookup_mpls_trace,
1231     .n_next_nodes = 0,
1232 };
1233 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf)
1234
1235 typedef enum lookup_ip_dst_mcast_next_t_ {
1236     LOOKUP_IP_DST_MCAST_NEXT_DROP,
1237     LOOKUP_IP_DST_MCAST_NEXT_RPF,
1238     LOOKUP_IP_DST_MCAST_N_NEXT,
1239 } mfib_forward_lookup_next_t;
1240
1241 always_inline uword
1242 lookup_dpo_ip_dst_mcast_inline (vlib_main_t * vm,
1243                                 vlib_node_runtime_t * node,
1244                                 vlib_frame_t * from_frame,
1245                                 int is_v4)
1246 {
1247     u32 n_left_from, next_index, * from, * to_next;
1248
1249     from = vlib_frame_vector_args (from_frame);
1250     n_left_from = from_frame->n_vectors;
1251
1252     next_index = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1253
1254     while (n_left_from > 0)
1255     {
1256         u32 n_left_to_next;
1257
1258         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
1259
1260         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1261         /*   } */
1262
1263         while (n_left_from > 0 && n_left_to_next > 0)
1264         {
1265             u32 bi0, lkdi0, fib_index0,  next0;
1266             const lookup_dpo_t * lkd0;
1267             fib_node_index_t mfei0;
1268             vlib_buffer_t * b0;
1269
1270             bi0 = from[0];
1271             to_next[0] = bi0;
1272             from += 1;
1273             to_next += 1;
1274             n_left_from -= 1;
1275             n_left_to_next -= 1;
1276
1277             b0 = vlib_get_buffer (vm, bi0);
1278
1279             /* dst lookup was done by mpls lookup */
1280             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1281             lkd0 = lookup_dpo_get(lkdi0);
1282             fib_index0 = lkd0->lkd_fib_index;
1283             next0 = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1284
1285             if (is_v4)
1286             {
1287                 ip4_header_t * ip0;
1288
1289                 ip0 = vlib_buffer_get_current (b0);
1290                 mfei0 = ip4_mfib_table_lookup(ip4_mfib_get(fib_index0),
1291                                               &ip0->src_address,
1292                                               &ip0->dst_address,
1293                                               64);
1294                 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1295                 {
1296                     lookup_trace_t *tr = vlib_add_trace (vm, node,
1297                                                          b0, sizeof (*tr));
1298                     tr->fib_index = fib_index0;
1299                     tr->lbi = mfei0;
1300                     tr->addr.ip4 = ip0->dst_address;
1301                 }
1302             }
1303             else
1304             {
1305                 ip6_header_t * ip0;
1306
1307                 ip0 = vlib_buffer_get_current (b0);
1308                 mfei0 = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index0),
1309                                                &ip0->src_address,
1310                                                &ip0->dst_address);
1311                 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1312                 {
1313                     lookup_trace_t *tr = vlib_add_trace (vm, node,
1314                                                          b0, sizeof (*tr));
1315                     tr->fib_index = fib_index0;
1316                     tr->lbi = mfei0;
1317                     tr->addr.ip6 = ip0->dst_address;
1318                 }
1319             }
1320
1321             vnet_buffer (b0)->ip.adj_index[VLIB_TX] = mfei0;
1322
1323             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
1324                 vnet_buffer2(b0)->loop_counter = 0;
1325                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
1326             }
1327
1328             vnet_buffer2(b0)->loop_counter++;
1329
1330             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
1331                 next0 = LOOKUP_IP_DST_MCAST_NEXT_DROP;
1332
1333             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1334                                             n_left_to_next, bi0, next0);
1335         }
1336         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1337     }
1338     return from_frame->n_vectors;
1339 }
1340
1341 static uword
1342 lookup_ip4_dst_mcast (vlib_main_t * vm,
1343                       vlib_node_runtime_t * node,
1344                       vlib_frame_t * from_frame)
1345 {
1346     return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 1));
1347 }
1348
1349 VLIB_REGISTER_NODE (lookup_ip4_dst_mcast_node) = {
1350     .function = lookup_ip4_dst_mcast,
1351     .name = "lookup-ip4-dst-mcast",
1352     .vector_size = sizeof (u32),
1353
1354     .format_trace = format_lookup_trace,
1355     .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1356     .next_nodes = {
1357         [LOOKUP_IP_DST_MCAST_NEXT_DROP] = "ip4-drop",
1358         [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip4-mfib-forward-rpf",
1359     },
1360 };
1361 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_mcast_node,
1362                               lookup_ip4_dst_mcast)
1363
1364 static uword
1365 lookup_ip6_dst_mcast (vlib_main_t * vm,
1366                       vlib_node_runtime_t * node,
1367                       vlib_frame_t * from_frame)
1368 {
1369     return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 0));
1370 }
1371
1372 VLIB_REGISTER_NODE (lookup_ip6_dst_mcast_node) = {
1373     .function = lookup_ip6_dst_mcast,
1374     .name = "lookup-ip6-dst-mcast",
1375     .vector_size = sizeof (u32),
1376
1377     .format_trace = format_lookup_trace,
1378     .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1379     .next_nodes = {
1380         [LOOKUP_IP_DST_MCAST_NEXT_DROP] = "ip6-drop",
1381         [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip6-mfib-forward-rpf",
1382     },
1383 };
1384 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_mcast_node,
1385                               lookup_ip6_dst_mcast)
1386
1387 static void
1388 lookup_dpo_mem_show (void)
1389 {
1390     fib_show_memory_usage("Lookup",
1391                           pool_elts(lookup_dpo_pool),
1392                           pool_len(lookup_dpo_pool),
1393                           sizeof(lookup_dpo_t));
1394 }
1395
1396 const static dpo_vft_t lkd_vft = {
1397     .dv_lock = lookup_dpo_lock,
1398     .dv_unlock = lookup_dpo_unlock,
1399     .dv_format = format_lookup_dpo,
1400 };
1401 const static dpo_vft_t lkd_vft_w_mem_show = {
1402     .dv_lock = lookup_dpo_lock,
1403     .dv_unlock = lookup_dpo_unlock,
1404     .dv_format = format_lookup_dpo,
1405     .dv_mem_show = lookup_dpo_mem_show,
1406 };
1407
1408 const static char* const lookup_src_ip4_nodes[] =
1409 {
1410     "lookup-ip4-src",
1411     NULL,
1412 };
1413 const static char* const lookup_src_ip6_nodes[] =
1414 {
1415     "lookup-ip6-src",
1416     NULL,
1417 };
1418 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
1419 {
1420     [DPO_PROTO_IP4]  = lookup_src_ip4_nodes,
1421     [DPO_PROTO_IP6]  = lookup_src_ip6_nodes,
1422     [DPO_PROTO_MPLS] = NULL,
1423 };
1424
1425 const static char* const lookup_dst_ip4_nodes[] =
1426 {
1427     "lookup-ip4-dst",
1428     NULL,
1429 };
1430 const static char* const lookup_dst_ip6_nodes[] =
1431 {
1432     "lookup-ip6-dst",
1433     NULL,
1434 };
1435 const static char* const lookup_dst_mpls_nodes[] =
1436 {
1437     "lookup-mpls-dst",
1438     NULL,
1439 };
1440 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
1441 {
1442     [DPO_PROTO_IP4]  = lookup_dst_ip4_nodes,
1443     [DPO_PROTO_IP6]  = lookup_dst_ip6_nodes,
1444     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
1445 };
1446
1447 const static char* const lookup_dst_mcast_ip4_nodes[] =
1448 {
1449     "lookup-ip4-dst-mcast",
1450     NULL,
1451 };
1452 const static char* const lookup_dst_mcast_ip6_nodes[] =
1453 {
1454     "lookup-ip6-dst-mcast",
1455     NULL,
1456 };
1457 const static char* const * const lookup_dst_mcast_nodes[DPO_PROTO_NUM] =
1458 {
1459     [DPO_PROTO_IP4]  = lookup_dst_mcast_ip4_nodes,
1460     [DPO_PROTO_IP6]  = lookup_dst_mcast_ip6_nodes,
1461 };
1462
1463 const static char* const lookup_dst_from_interface_ip4_nodes[] =
1464 {
1465     "lookup-ip4-dst-itf",
1466     NULL,
1467 };
1468 const static char* const lookup_dst_from_interface_ip6_nodes[] =
1469 {
1470     "lookup-ip6-dst-itf",
1471     NULL,
1472 };
1473 const static char* const lookup_dst_from_interface_mpls_nodes[] =
1474 {
1475     "lookup-mpls-dst-itf",
1476     NULL,
1477 };
1478 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
1479 {
1480     [DPO_PROTO_IP4]  = lookup_dst_from_interface_ip4_nodes,
1481     [DPO_PROTO_IP6]  = lookup_dst_from_interface_ip6_nodes,
1482     [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
1483 };
1484
1485
1486 void
1487 lookup_dpo_module_init (void)
1488 {
1489     dpo_register(DPO_LOOKUP, &lkd_vft_w_mem_show, NULL);
1490
1491     /*
1492      * There are various sorts of lookup; src or dst addr v4 /v6 etc.
1493      * there isn't an object type for each (there is only the lookup_dpo_t),
1494      * but, for performance reasons, there is a data plane function, and hence
1495      * VLIB node for each. VLIB graph node construction is based on DPO types
1496      * so we create sub-types.
1497      */
1498     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
1499         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
1500     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
1501         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
1502     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST] =
1503         dpo_register_new_type(&lkd_vft, lookup_dst_mcast_nodes);
1504     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
1505         dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes);
1506 }