src/vnet/dpo/lookup_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/lookup_dpo.h>
  18 #include <vnet/dpo/load_balance_map.h>
  19 #include <vnet/mpls/mpls_lookup.h>
  20 #include <vnet/fib/fib_table.h>
  21 #include <vnet/fib/ip4_fib.h>
  22 #include <vnet/fib/ip6_fib.h>
  23 #include <vnet/fib/mpls_fib.h>
  24 #include <vnet/mfib/mfib_table.h>
  25 #include <vnet/mfib/ip4_mfib.h>
  26 #include <vnet/mfib/ip6_mfib.h>
  27
  28 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
  29 static const char *const lookup_cast_names[] = LOOKUP_CASTS;
  30
  31 /**
  32  * If a packet encounters a lookup DPO more than the many times
  33  * then we assume there is a loop in the forward graph and drop the packet
  34  */
  35 #define MAX_LUKPS_PER_PACKET 4
  36
  37 /**
  38  * @brief Enumeration of the lookup subtypes
  39  */
  40 typedef enum lookup_sub_type_t_
  41 {
  42     LOOKUP_SUB_TYPE_SRC,
  43     LOOKUP_SUB_TYPE_DST,
  44     LOOKUP_SUB_TYPE_DST_MCAST,
  45     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
  46 } lookup_sub_type_t;
  47 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
  48
  49 #define FOR_EACH_LOOKUP_SUB_TYPE(_st)                                   \
  50     for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
  51
  52 /**
  53  * @brief pool of all MPLS Label DPOs
  54  */
  55 lookup_dpo_t *lookup_dpo_pool;
  56
  57 /**
  58  * @brief An array of registered DPO type values for the sub-types
  59  */
  60 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
  61
  62 static lookup_dpo_t *
  63 lookup_dpo_alloc (void)
  64 {
  65     lookup_dpo_t *lkd;
  66
  67     pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
  68
  69     return (lkd);
  70 }
  71
  72 static index_t
  73 lookup_dpo_get_index (lookup_dpo_t *lkd)
  74 {
  75     return (lkd - lookup_dpo_pool);
  76 }
  77
  78 static void
  79 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
  80                           dpo_proto_t proto,
  81                           lookup_cast_t cast,
  82                           lookup_input_t input,
  83                           lookup_table_t table_config,
  84                           dpo_id_t *dpo)
  85 {
  86     lookup_dpo_t *lkd;
  87     dpo_type_t type;
  88
  89     lkd = lookup_dpo_alloc();
  90     lkd->lkd_fib_index = fib_index;
  91     lkd->lkd_proto = proto;
  92     lkd->lkd_input = input;
  93     lkd->lkd_table = table_config;
  94     lkd->lkd_cast  = cast;
  95
  96     /*
  97      * use the input type to select the lookup sub-type
  98      */
  99     type = 0;
 100
 101     switch (input)
 102     {
 103     case LOOKUP_INPUT_SRC_ADDR:
 104         type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
 105         break;
 106     case LOOKUP_INPUT_DST_ADDR:
 107         switch (table_config)
 108         {
 109         case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
 110             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
 111             break;
 112         case LOOKUP_TABLE_FROM_CONFIG:
 113             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
 114             break;
 115         }
 116         if (LOOKUP_MULTICAST == cast)
 117         {
 118             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST];
 119         }
 120     }
 121
 122     if (0 == type)
 123     {
 124         dpo_reset(dpo);
 125     }
 126     else
 127     {
 128         dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
 129     }
 130 }
 131
 132 void
 133 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
 134                                     dpo_proto_t proto,
 135                                     lookup_cast_t cast,
 136                                     lookup_input_t input,
 137                                     lookup_table_t table_config,
 138                                     dpo_id_t *dpo)
 139 {
 140     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 141     {
 142         if (LOOKUP_UNICAST == cast)
 143         {
 144             fib_table_lock(fib_index,
 145                            dpo_proto_to_fib(proto),
 146                            FIB_SOURCE_RR);
 147         }
 148         else
 149         {
 150             mfib_table_lock(fib_index,
 151                             dpo_proto_to_fib(proto),
 152                             MFIB_SOURCE_RR);
 153         }
 154     }
 155     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 156 }
 157
 158 void
 159 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 160                                    dpo_proto_t proto,
 161                                    lookup_cast_t cast,
 162                                    lookup_input_t input,
 163                                    lookup_table_t table_config,
 164                                    dpo_id_t *dpo)
 165 {
 166     fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
 167
 168     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 169     {
 170         if (LOOKUP_UNICAST == cast)
 171         {
 172             fib_index =
 173                 fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 174                                                   table_id,
 175                                                   FIB_SOURCE_RR);
 176         }
 177         else
 178         {
 179             fib_index =
 180                 mfib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 181                                                    table_id,
 182                                                    MFIB_SOURCE_RR);
 183         }
 184     }
 185
 186     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
 187     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 188 }
 189
 190 u8*
 191 format_lookup_dpo (u8 *s, va_list *args)
 192 {
 193     index_t index = va_arg (*args, index_t);
 194     lookup_dpo_t *lkd;
 195
 196     lkd = lookup_dpo_get(index);
 197
 198     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
 199     {
 200         s = format(s, "%s,%s lookup in interface's %U table",
 201                    lookup_input_names[lkd->lkd_input],
 202                    lookup_cast_names[lkd->lkd_cast],
 203                    format_dpo_proto, lkd->lkd_proto);
 204     }
 205     else
 206     {
 207         if (LOOKUP_UNICAST == lkd->lkd_cast)
 208         {
 209             s = format(s, "%s,%s lookup in %U",
 210                        lookup_input_names[lkd->lkd_input],
 211                        lookup_cast_names[lkd->lkd_cast],
 212                        format_fib_table_name, lkd->lkd_fib_index,
 213                        dpo_proto_to_fib(lkd->lkd_proto));
 214         }
 215         else
 216         {
 217             s = format(s, "%s,%s lookup in %U",
 218                        lookup_input_names[lkd->lkd_input],
 219                        lookup_cast_names[lkd->lkd_cast],
 220                        format_mfib_table_name, lkd->lkd_fib_index,
 221                        dpo_proto_to_fib(lkd->lkd_proto));
 222         }
 223     }
 224     return (s);
 225 }
 226
 227 static void
 228 lookup_dpo_lock (dpo_id_t *dpo)
 229 {
 230     lookup_dpo_t *lkd;
 231
 232     lkd = lookup_dpo_get(dpo->dpoi_index);
 233
 234     lkd->lkd_locks++;
 235 }
 236
 237 static void
 238 lookup_dpo_unlock (dpo_id_t *dpo)
 239 {
 240     lookup_dpo_t *lkd;
 241
 242     lkd = lookup_dpo_get(dpo->dpoi_index);
 243
 244     lkd->lkd_locks--;
 245
 246     if (0 == lkd->lkd_locks)
 247     {
 248         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
 249         {
 250             if (LOOKUP_UNICAST == lkd->lkd_cast)
 251             {
 252                 fib_table_unlock(lkd->lkd_fib_index,
 253                                  dpo_proto_to_fib(lkd->lkd_proto),
 254                                  FIB_SOURCE_RR);
 255             }
 256             else
 257             {
 258                 mfib_table_unlock(lkd->lkd_fib_index,
 259                                   dpo_proto_to_fib(lkd->lkd_proto),
 260                                   MFIB_SOURCE_RR);
 261             }
 262         }
 263         pool_put(lookup_dpo_pool, lkd);
 264     }
 265 }
 266
 267 always_inline void
 268 ip4_src_fib_lookup_one (u32 src_fib_index0,
 269                         const ip4_address_t * addr0,
 270                         u32 * src_adj_index0)
 271 {
 272     ip4_fib_mtrie_leaf_t leaf0;
 273     ip4_fib_mtrie_t * mtrie0;
 274
 275     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 276
 277     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 278     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 279     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 280
 281     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 282 }
 283
 284 always_inline void
 285 ip4_src_fib_lookup_two (u32 src_fib_index0,
 286                         u32 src_fib_index1,
 287                         const ip4_address_t * addr0,
 288                         const ip4_address_t * addr1,
 289                         u32 * src_adj_index0,
 290                         u32 * src_adj_index1)
 291 {
 292     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 293     ip4_fib_mtrie_t * mtrie0, * mtrie1;
 294
 295     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 296     mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
 297
 298     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 299     leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, addr1);
 300
 301     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 302     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
 303
 304     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 305     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
 306
 307     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 308     src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 309 }
 310
 311 /**
 312  * @brief Lookup trace  data
 313  */
 314 typedef struct lookup_trace_t_
 315 {
 316     union {
 317         ip46_address_t addr;
 318         mpls_unicast_header_t hdr;
 319     };
 320     fib_node_index_t fib_index;
 321     index_t lbi;
 322 } lookup_trace_t;
 323
 324
 325 always_inline uword
 326 lookup_dpo_ip4_inline (vlib_main_t * vm,
 327                        vlib_node_runtime_t * node,
 328                        vlib_frame_t * from_frame,
 329                        int input_src_addr,
 330                        int table_from_interface)
 331 {
 332     u32 n_left_from, next_index, * from, * to_next;
 333     u32 thread_index = vlib_get_thread_index();
 334     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 335
 336     from = vlib_frame_vector_args (from_frame);
 337     n_left_from = from_frame->n_vectors;
 338
 339     next_index = node->cached_next_index;
 340
 341     while (n_left_from > 0)
 342     {
 343         u32 n_left_to_next;
 344
 345         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 346
 347         while (n_left_from >= 4 && n_left_to_next > 2)
 348         {
 349             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 350             flow_hash_config_t flow_hash_config0;
 351             const ip4_address_t *input_addr0;
 352             const load_balance_t *lb0;
 353             const lookup_dpo_t * lkd0;
 354             const ip4_header_t * ip0;
 355             const dpo_id_t *dpo0;
 356             vlib_buffer_t * b0;
 357             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 358             flow_hash_config_t flow_hash_config1;
 359             const ip4_address_t *input_addr1;
 360             const load_balance_t *lb1;
 361             const lookup_dpo_t * lkd1;
 362             const ip4_header_t * ip1;
 363             const dpo_id_t *dpo1;
 364             vlib_buffer_t * b1;
 365
 366             /* Prefetch next iteration. */
 367             {
 368                 vlib_buffer_t * p2, * p3;
 369
 370                 p2 = vlib_get_buffer (vm, from[2]);
 371                 p3 = vlib_get_buffer (vm, from[3]);
 372
 373                 vlib_prefetch_buffer_header (p2, LOAD);
 374                 vlib_prefetch_buffer_header (p3, LOAD);
 375
 376                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 377                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 378             }
 379
 380             bi0 = from[0];
 381             to_next[0] = bi0;
 382             bi1 = from[1];
 383             to_next[1] = bi1;
 384             from += 2;
 385             to_next += 2;
 386             n_left_from -= 2;
 387             n_left_to_next -= 2;
 388
 389             b0 = vlib_get_buffer (vm, bi0);
 390             ip0 = vlib_buffer_get_current (b0);
 391             b1 = vlib_get_buffer (vm, bi1);
 392             ip1 = vlib_buffer_get_current (b1);
 393
 394             /* dst lookup was done by ip4 lookup */
 395             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 396             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 397             lkd0 = lookup_dpo_get(lkdi0);
 398             lkd1 = lookup_dpo_get(lkdi1);
 399
 400             /*
 401              * choose between a lookup using the fib index in the DPO
 402              * or getting the FIB index from the interface.
 403              */
 404             if (table_from_interface)
 405             {
 406                 fib_index0 =
 407                     ip4_fib_table_get_index_for_sw_if_index(
 408                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 409                 fib_index1 =
 410                     ip4_fib_table_get_index_for_sw_if_index(
 411                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 412             }
 413             else
 414             {
 415                 fib_index0 = lkd0->lkd_fib_index;
 416                 fib_index1 = lkd1->lkd_fib_index;
 417             }
 418
 419             /*
 420              * choose between a source or destination address lookup in the table
 421              */
 422             if (input_src_addr)
 423             {
 424                 input_addr0 = &ip0->src_address;
 425                 input_addr1 = &ip1->src_address;
 426             }
 427             else
 428             {
 429                 input_addr0 = &ip0->dst_address;
 430                 input_addr1 = &ip1->dst_address;
 431             }
 432
 433             /* do lookup */
 434             ip4_src_fib_lookup_two (fib_index0, fib_index1,
 435                                     input_addr0, input_addr1,
 436                                     &lbi0, &lbi1);
 437             lb0 = load_balance_get(lbi0);
 438             lb1 = load_balance_get(lbi1);
 439
 440             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 441             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 442
 443             /* Use flow hash to compute multipath adjacency. */
 444             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 445             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 446
 447             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 448             {
 449                 flow_hash_config0 = lb0->lb_hash_config;
 450                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 451                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 452             }
 453
 454             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 455             {
 456                 flow_hash_config1 = lb1->lb_hash_config;
 457                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 458                     ip4_compute_flow_hash (ip1, flow_hash_config1);
 459             }
 460
 461             dpo0 = load_balance_get_bucket_i(lb0,
 462                                              (hash_c0 &
 463                                               (lb0->lb_n_buckets_minus_1)));
 464             dpo1 = load_balance_get_bucket_i(lb1,
 465                                              (hash_c1 &
 466                                               (lb1->lb_n_buckets_minus_1)));
 467
 468             next0 = dpo0->dpoi_next_node;
 469             next1 = dpo1->dpoi_next_node;
 470             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 471             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 472
 473             vlib_increment_combined_counter
 474                 (cm, thread_index, lbi0, 1,
 475                  vlib_buffer_length_in_chain (vm, b0));
 476             vlib_increment_combined_counter
 477                 (cm, thread_index, lbi1, 1,
 478                  vlib_buffer_length_in_chain (vm, b1));
 479
 480             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 481                 vnet_buffer2(b0)->loop_counter = 0;
 482                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 483             }
 484             if (!(b1->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 485                 vnet_buffer2(b1)->loop_counter = 0;
 486                 b1->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 487             }
 488
 489             vnet_buffer2(b0)->loop_counter++;
 490             vnet_buffer2(b1)->loop_counter++;
 491
 492             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 493                 next0 = IP_LOOKUP_NEXT_DROP;
 494             if (PREDICT_FALSE(vnet_buffer2(b1)->loop_counter > MAX_LUKPS_PER_PACKET))
 495                 next1 = IP_LOOKUP_NEXT_DROP;
 496
 497             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 498             {
 499                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 500                                                      b0, sizeof (*tr));
 501                 tr->fib_index = fib_index0;
 502                 tr->lbi = lbi0;
 503                 tr->addr.ip4 = *input_addr0;
 504             }
 505             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 506             {
 507                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 508                                                      b1, sizeof (*tr));
 509                 tr->fib_index = fib_index1;
 510                 tr->lbi = lbi1;
 511                 tr->addr.ip4 = *input_addr1;
 512             }
 513
 514             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 515                                              to_next, n_left_to_next,
 516                                              bi0, bi1, next0, next1);
 517         }
 518
 519         while (n_left_from > 0 && n_left_to_next > 0)
 520         {
 521             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 522             flow_hash_config_t flow_hash_config0;
 523             const ip4_address_t *input_addr;
 524             const load_balance_t *lb0;
 525             const lookup_dpo_t * lkd0;
 526             const ip4_header_t * ip0;
 527             const dpo_id_t *dpo0;
 528             vlib_buffer_t * b0;
 529
 530             bi0 = from[0];
 531             to_next[0] = bi0;
 532             from += 1;
 533             to_next += 1;
 534             n_left_from -= 1;
 535             n_left_to_next -= 1;
 536
 537             b0 = vlib_get_buffer (vm, bi0);
 538             ip0 = vlib_buffer_get_current (b0);
 539
 540             /* dst lookup was done by ip4 lookup */
 541             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 542             lkd0 = lookup_dpo_get(lkdi0);
 543
 544             /*
 545              * choose between a lookup using the fib index in the DPO
 546              * or getting the FIB index from the interface.
 547              */
 548             if (table_from_interface)
 549             {
 550                 fib_index0 =
 551                     ip4_fib_table_get_index_for_sw_if_index(
 552                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 553             }
 554             else
 555             {
 556                 fib_index0 = lkd0->lkd_fib_index;
 557             }
 558
 559             /*
 560              * choose between a source or destination address lookup in the table
 561              */
 562             if (input_src_addr)
 563             {
 564                 input_addr = &ip0->src_address;
 565             }
 566             else
 567             {
 568                 input_addr = &ip0->dst_address;
 569             }
 570
 571             /* do lookup */
 572             ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
 573             lb0 = load_balance_get(lbi0);
 574
 575             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 576
 577             /* Use flow hash to compute multipath adjacency. */
 578             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 579
 580             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 581             {
 582                 flow_hash_config0 = lb0->lb_hash_config;
 583                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 584                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 585             }
 586
 587             dpo0 = load_balance_get_bucket_i(lb0,
 588                                              (hash_c0 &
 589                                               (lb0->lb_n_buckets_minus_1)));
 590
 591             next0 = dpo0->dpoi_next_node;
 592             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 593
 594             vlib_increment_combined_counter
 595                 (cm, thread_index, lbi0, 1,
 596                  vlib_buffer_length_in_chain (vm, b0));
 597
 598             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 599                 vnet_buffer2(b0)->loop_counter = 0;
 600                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 601             }
 602
 603             vnet_buffer2(b0)->loop_counter++;
 604
 605             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 606                 next0 = IP_LOOKUP_NEXT_DROP;
 607
 608             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 609             {
 610                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 611                                                      b0, sizeof (*tr));
 612                 tr->fib_index = fib_index0;
 613                 tr->lbi = lbi0;
 614                 tr->addr.ip4 = *input_addr;
 615             }
 616
 617             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 618                                             n_left_to_next, bi0, next0);
 619         }
 620         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 621     }
 622     return from_frame->n_vectors;
 623 }
 624
 625 static u8 *
 626 format_lookup_trace (u8 * s, va_list * args)
 627 {
 628     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 629     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 630     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 631     u32 indent = format_get_indent (s);
 632     s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
 633                 format_white_space, indent,
 634                 t->fib_index,
 635                 format_ip46_address, &t->addr, IP46_TYPE_ANY,
 636                 t->lbi);
 637     return s;
 638 }
 639
 640 VLIB_NODE_FN (lookup_ip4_dst_node) (vlib_main_t * vm,
 641                 vlib_node_runtime_t * node,
 642                 vlib_frame_t * from_frame)
 643 {
 644     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
 645 }
 646
 647 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
 648     .name = "lookup-ip4-dst",
 649     .vector_size = sizeof (u32),
 650     .sibling_of = "ip4-lookup",
 651     .format_trace = format_lookup_trace,
 652 };
 653
 654 VLIB_NODE_FN (lookup_ip4_dst_itf_node) (vlib_main_t * vm,
 655                     vlib_node_runtime_t * node,
 656                     vlib_frame_t * from_frame)
 657 {
 658     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
 659 }
 660
 661 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
 662     .name = "lookup-ip4-dst-itf",
 663     .vector_size = sizeof (u32),
 664     .sibling_of = "ip4-lookup",
 665     .format_trace = format_lookup_trace,
 666 };
 667
 668 VLIB_NODE_FN (lookup_ip4_src_node) (vlib_main_t * vm,
 669                 vlib_node_runtime_t * node,
 670                 vlib_frame_t * from_frame)
 671 {
 672     return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
 673 }
 674
 675 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
 676     .name = "lookup-ip4-src",
 677     .vector_size = sizeof (u32),
 678     .format_trace = format_lookup_trace,
 679     .sibling_of = "ip4-lookup",
 680 };
 681
 682 always_inline uword
 683 lookup_dpo_ip6_inline (vlib_main_t * vm,
 684                        vlib_node_runtime_t * node,
 685                        vlib_frame_t * from_frame,
 686                        int input_src_addr,
 687                        int table_from_interface)
 688 {
 689     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 690     u32 n_left_from, next_index, * from, * to_next;
 691     u32 thread_index = vlib_get_thread_index();
 692
 693     from = vlib_frame_vector_args (from_frame);
 694     n_left_from = from_frame->n_vectors;
 695
 696     next_index = node->cached_next_index;
 697
 698     while (n_left_from > 0)
 699     {
 700         u32 n_left_to_next;
 701
 702         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 703
 704         while (n_left_from >= 4 && n_left_to_next > 2)
 705         {
 706             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 707             flow_hash_config_t flow_hash_config0;
 708             const ip6_address_t *input_addr0;
 709             const load_balance_t *lb0;
 710             const lookup_dpo_t * lkd0;
 711             const ip6_header_t * ip0;
 712             const dpo_id_t *dpo0;
 713             vlib_buffer_t * b0;
 714             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 715             flow_hash_config_t flow_hash_config1;
 716             const ip6_address_t *input_addr1;
 717             const load_balance_t *lb1;
 718             const lookup_dpo_t * lkd1;
 719             const ip6_header_t * ip1;
 720             const dpo_id_t *dpo1;
 721             vlib_buffer_t * b1;
 722
 723             /* Prefetch next iteration. */
 724             {
 725                 vlib_buffer_t * p2, * p3;
 726
 727                 p2 = vlib_get_buffer (vm, from[2]);
 728                 p3 = vlib_get_buffer (vm, from[3]);
 729
 730                 vlib_prefetch_buffer_header (p2, LOAD);
 731                 vlib_prefetch_buffer_header (p3, LOAD);
 732
 733                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 734                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 735             }
 736
 737             bi0 = from[0];
 738             to_next[0] = bi0;
 739             bi1 = from[1];
 740             to_next[1] = bi1;
 741             from += 2;
 742             to_next += 2;
 743             n_left_from -= 2;
 744             n_left_to_next -= 2;
 745
 746             b0 = vlib_get_buffer (vm, bi0);
 747             ip0 = vlib_buffer_get_current (b0);
 748             b1 = vlib_get_buffer (vm, bi1);
 749             ip1 = vlib_buffer_get_current (b1);
 750
 751             /* dst lookup was done by ip6 lookup */
 752             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 753             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 754             lkd0 = lookup_dpo_get(lkdi0);
 755             lkd1 = lookup_dpo_get(lkdi1);
 756
 757             /*
 758              * choose between a lookup using the fib index in the DPO
 759              * or getting the FIB index from the interface.
 760              */
 761             if (table_from_interface)
 762             {
 763                 fib_index0 =
 764                     ip6_fib_table_get_index_for_sw_if_index(
 765                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 766                 fib_index1 =
 767                     ip6_fib_table_get_index_for_sw_if_index(
 768                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 769             }
 770             else
 771             {
 772                 fib_index0 = lkd0->lkd_fib_index;
 773                 fib_index1 = lkd1->lkd_fib_index;
 774             }
 775
 776             /*
 777              * choose between a source or destination address lookup in the table
 778              */
 779             if (input_src_addr)
 780             {
 781                 input_addr0 = &ip0->src_address;
 782                 input_addr1 = &ip1->src_address;
 783             }
 784             else
 785             {
 786                 input_addr0 = &ip0->dst_address;
 787                 input_addr1 = &ip1->dst_address;
 788             }
 789
 790             /* do src lookup */
 791             lbi0 = ip6_fib_table_fwding_lookup(
 792                                                fib_index0,
 793                                                input_addr0);
 794             lbi1 = ip6_fib_table_fwding_lookup(
 795                                                fib_index1,
 796                                                input_addr1);
 797             lb0 = load_balance_get(lbi0);
 798             lb1 = load_balance_get(lbi1);
 799
 800             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 801             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 802
 803             /* Use flow hash to compute multipath adjacency. */
 804             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 805             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 806
 807             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 808                 vnet_buffer2(b0)->loop_counter = 0;
 809                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 810             }
 811             if (!(b1->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 812                 vnet_buffer2(b1)->loop_counter = 0;
 813                 b1->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 814             }
 815
 816             vnet_buffer2(b0)->loop_counter++;
 817             vnet_buffer2(b1)->loop_counter++;
 818
 819             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 820                 next0 = IP_LOOKUP_NEXT_DROP;
 821             if (PREDICT_FALSE(vnet_buffer2(b1)->loop_counter > MAX_LUKPS_PER_PACKET))
 822                 next1 = IP_LOOKUP_NEXT_DROP;
 823
 824             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 825             {
 826                 flow_hash_config0 = lb0->lb_hash_config;
 827                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 828                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 829             }
 830
 831             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 832             {
 833                 flow_hash_config1 = lb1->lb_hash_config;
 834                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 835                     ip6_compute_flow_hash (ip1, flow_hash_config1);
 836             }
 837
 838             dpo0 = load_balance_get_bucket_i(lb0,
 839                                              (hash_c0 &
 840                                               (lb0->lb_n_buckets_minus_1)));
 841             dpo1 = load_balance_get_bucket_i(lb1,
 842                                              (hash_c1 &
 843                                               (lb1->lb_n_buckets_minus_1)));
 844
 845             next0 = dpo0->dpoi_next_node;
 846             next1 = dpo1->dpoi_next_node;
 847             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 848             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 849
 850             vlib_increment_combined_counter
 851                 (cm, thread_index, lbi0, 1,
 852                  vlib_buffer_length_in_chain (vm, b0));
 853             vlib_increment_combined_counter
 854                 (cm, thread_index, lbi1, 1,
 855                  vlib_buffer_length_in_chain (vm, b1));
 856
 857             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 858             {
 859                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 860                                                      b0, sizeof (*tr));
 861                 tr->fib_index = fib_index0;
 862                 tr->lbi = lbi0;
 863                 tr->addr.ip6 = *input_addr0;
 864             }
 865             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 866             {
 867                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 868                                                      b1, sizeof (*tr));
 869                 tr->fib_index = fib_index1;
 870                 tr->lbi = lbi1;
 871                 tr->addr.ip6 = *input_addr1;
 872             }
 873             vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
 874                                             n_left_to_next, bi0, bi1,
 875                                             next0, next1);
 876         }
 877         while (n_left_from > 0 && n_left_to_next > 0)
 878         {
 879             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 880             flow_hash_config_t flow_hash_config0;
 881             const ip6_address_t *input_addr0;
 882             const load_balance_t *lb0;
 883             const lookup_dpo_t * lkd0;
 884             const ip6_header_t * ip0;
 885             const dpo_id_t *dpo0;
 886             vlib_buffer_t * b0;
 887
 888             bi0 = from[0];
 889             to_next[0] = bi0;
 890             from += 1;
 891             to_next += 1;
 892             n_left_from -= 1;
 893             n_left_to_next -= 1;
 894
 895             b0 = vlib_get_buffer (vm, bi0);
 896             ip0 = vlib_buffer_get_current (b0);
 897
 898             /* dst lookup was done by ip6 lookup */
 899             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 900             lkd0 = lookup_dpo_get(lkdi0);
 901
 902             /*
 903              * choose between a lookup using the fib index in the DPO
 904              * or getting the FIB index from the interface.
 905              */
 906             if (table_from_interface)
 907             {
 908                 fib_index0 =
 909                     ip6_fib_table_get_index_for_sw_if_index(
 910                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 911             }
 912             else
 913             {
 914                 fib_index0 = lkd0->lkd_fib_index;
 915             }
 916
 917             /*
 918              * choose between a source or destination address lookup in the table
 919              */
 920             if (input_src_addr)
 921             {
 922                 input_addr0 = &ip0->src_address;
 923             }
 924             else
 925             {
 926                 input_addr0 = &ip0->dst_address;
 927             }
 928
 929             /* do src lookup */
 930             lbi0 = ip6_fib_table_fwding_lookup(
 931                                                fib_index0,
 932                                                input_addr0);
 933             lb0 = load_balance_get(lbi0);
 934
 935             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 936
 937             /* Use flow hash to compute multipath adjacency. */
 938             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 939
 940             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 941             {
 942                 flow_hash_config0 = lb0->lb_hash_config;
 943                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 944                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 945             }
 946
 947             dpo0 = load_balance_get_bucket_i(lb0,
 948                                              (hash_c0 &
 949                                               (lb0->lb_n_buckets_minus_1)));
 950
 951             next0 = dpo0->dpoi_next_node;
 952             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 953
 954             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 955                 vnet_buffer2(b0)->loop_counter = 0;
 956                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 957             }
 958
 959             vnet_buffer2(b0)->loop_counter++;
 960
 961             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 962                 next0 = IP_LOOKUP_NEXT_DROP;
 963
 964             vlib_increment_combined_counter
 965                 (cm, thread_index, lbi0, 1,
 966                  vlib_buffer_length_in_chain (vm, b0));
 967
 968             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 969             {
 970                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 971                                                      b0, sizeof (*tr));
 972                 tr->fib_index = fib_index0;
 973                 tr->lbi = lbi0;
 974                 tr->addr.ip6 = *input_addr0;
 975             }
 976             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 977                                             n_left_to_next, bi0, next0);
 978         }
 979         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 980     }
 981     return from_frame->n_vectors;
 982 }
 983
 984 VLIB_NODE_FN (lookup_ip6_dst_node) (vlib_main_t * vm,
 985                 vlib_node_runtime_t * node,
 986                 vlib_frame_t * from_frame)
 987 {
 988     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 0));
 989 }
 990
 991 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
 992     .name = "lookup-ip6-dst",
 993     .vector_size = sizeof (u32),
 994     .format_trace = format_lookup_trace,
 995     .sibling_of = "ip6-lookup",
 996 };
 997
 998 VLIB_NODE_FN (lookup_ip6_dst_itf_node) (vlib_main_t * vm,
 999                     vlib_node_runtime_t * node,
1000                     vlib_frame_t * from_frame)
1001 {
1002     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 1));
1003 }
1004
1005 VLIB_REGISTER_NODE (lookup_ip6_dst_itf_node) = {
1006     .name = "lookup-ip6-dst-itf",
1007     .vector_size = sizeof (u32),
1008     .format_trace = format_lookup_trace,
1009     .sibling_of = "ip6-lookup",
1010 };
1011
1012 VLIB_NODE_FN (lookup_ip6_src_node) (vlib_main_t * vm,
1013                 vlib_node_runtime_t * node,
1014                 vlib_frame_t * from_frame)
1015 {
1016     return (lookup_dpo_ip6_inline(vm, node, from_frame, 1, 0));
1017 }
1018
1019 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
1020     .name = "lookup-ip6-src",
1021     .vector_size = sizeof (u32),
1022     .format_trace = format_lookup_trace,
1023     .sibling_of = "ip6-lookup",
1024 };
1025
1026 always_inline uword
1027 lookup_dpo_mpls_inline (vlib_main_t * vm,
1028                        vlib_node_runtime_t * node,
1029                        vlib_frame_t * from_frame,
1030                        int table_from_interface)
1031 {
1032     u32 n_left_from, next_index, * from, * to_next;
1033     u32 thread_index = vlib_get_thread_index();
1034     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
1035
1036     from = vlib_frame_vector_args (from_frame);
1037     n_left_from = from_frame->n_vectors;
1038
1039     next_index = node->cached_next_index;
1040
1041     while (n_left_from > 0)
1042     {
1043         u32 n_left_to_next;
1044
1045         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
1046
1047         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1048         /*   } */
1049
1050         while (n_left_from > 0 && n_left_to_next > 0)
1051         {
1052             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash0;
1053             const mpls_unicast_header_t * hdr0;
1054             const load_balance_t *lb0;
1055             const lookup_dpo_t * lkd0;
1056             const dpo_id_t *dpo0;
1057             vlib_buffer_t * b0;
1058
1059             bi0 = from[0];
1060             to_next[0] = bi0;
1061             from += 1;
1062             to_next += 1;
1063             n_left_from -= 1;
1064             n_left_to_next -= 1;
1065
1066             b0 = vlib_get_buffer (vm, bi0);
1067             hdr0 = vlib_buffer_get_current (b0);
1068
1069             /* dst lookup was done by mpls lookup */
1070             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1071             lkd0 = lookup_dpo_get(lkdi0);
1072
1073             /*
1074              * choose between a lookup using the fib index in the DPO
1075              * or getting the FIB index from the interface.
1076              */
1077             if (table_from_interface)
1078             {
1079                 fib_index0 =
1080                     mpls_fib_table_get_index_for_sw_if_index(
1081                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1082             }
1083             else
1084             {
1085                 fib_index0 = lkd0->lkd_fib_index;
1086             }
1087
1088             /* do lookup */
1089             lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
1090             lb0  = load_balance_get(lbi0);
1091             dpo0 = load_balance_get_bucket_i(lb0, 0);
1092
1093             next0 = dpo0->dpoi_next_node;
1094             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1095
1096
1097             if (MPLS_IS_REPLICATE & lbi0)
1098             {
1099                 next0 = mpls_lookup_to_replicate_edge;
1100                 vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
1101                     (lbi0 & ~MPLS_IS_REPLICATE);
1102             }
1103             else
1104             {
1105                 lb0 = load_balance_get(lbi0);
1106                 ASSERT (lb0->lb_n_buckets > 0);
1107                 ASSERT (is_pow2 (lb0->lb_n_buckets));
1108
1109                 if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
1110                 {
1111                     hash0 = vnet_buffer (b0)->ip.flow_hash =
1112                         mpls_compute_flow_hash(hdr0, lb0->lb_hash_config);
1113                     dpo0 = load_balance_get_fwd_bucket
1114                         (lb0,
1115                          (hash0 & (lb0->lb_n_buckets_minus_1)));
1116                 }
1117                 else
1118                 {
1119                     dpo0 = load_balance_get_bucket_i (lb0, 0);
1120                 }
1121                 next0 = dpo0->dpoi_next_node;
1122
1123                 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1124
1125                 vlib_increment_combined_counter
1126                     (cm, thread_index, lbi0, 1,
1127                      vlib_buffer_length_in_chain (vm, b0));
1128             }
1129
1130             vnet_buffer (b0)->mpls.ttl = ((char*)hdr0)[3];
1131             vnet_buffer (b0)->mpls.exp = (((char*)hdr0)[2] & 0xe) >> 1;
1132             vnet_buffer (b0)->mpls.first = 1;
1133             vlib_buffer_advance(b0, sizeof(*hdr0));
1134
1135             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
1136                 vnet_buffer2(b0)->loop_counter = 0;
1137                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
1138             }
1139
1140             vnet_buffer2(b0)->loop_counter++;
1141
1142             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
1143                 next0 = MPLS_LOOKUP_NEXT_DROP;
1144
1145             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1146             {
1147                 lookup_trace_t *tr = vlib_add_trace (vm, node,
1148                                                      b0, sizeof (*tr));
1149                 tr->fib_index = fib_index0;
1150                 tr->lbi = lbi0;
1151                 tr->hdr = *hdr0;
1152             }
1153
1154            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1155                                             n_left_to_next, bi0, next0);
1156         }
1157         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1158     }
1159     return from_frame->n_vectors;
1160 }
1161
1162 static u8 *
1163 format_lookup_mpls_trace (u8 * s, va_list * args)
1164 {
1165     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1166     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1167     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
1168     u32 indent = format_get_indent (s);
1169     mpls_unicast_header_t hdr;
1170
1171     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
1172
1173     s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
1174                 format_white_space, indent,
1175                 t->fib_index,
1176                 format_mpls_header, hdr,
1177                 t->lbi);
1178     return s;
1179 }
1180
1181 VLIB_NODE_FN (lookup_mpls_dst_node) (vlib_main_t * vm,
1182                 vlib_node_runtime_t * node,
1183                 vlib_frame_t * from_frame)
1184 {
1185     return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
1186 }
1187
1188 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
1189     .name = "lookup-mpls-dst",
1190     .vector_size = sizeof (u32),
1191     .sibling_of = "mpls-lookup",
1192     .format_trace = format_lookup_mpls_trace,
1193     .n_next_nodes = 0,
1194 };
1195
1196 VLIB_NODE_FN (lookup_mpls_dst_itf_node) (vlib_main_t * vm,
1197                     vlib_node_runtime_t * node,
1198                     vlib_frame_t * from_frame)
1199 {
1200     return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
1201 }
1202
1203 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
1204     .name = "lookup-mpls-dst-itf",
1205     .vector_size = sizeof (u32),
1206     .sibling_of = "mpls-lookup",
1207     .format_trace = format_lookup_mpls_trace,
1208     .n_next_nodes = 0,
1209 };
1210
1211 typedef enum lookup_ip_dst_mcast_next_t_ {
1212     LOOKUP_IP_DST_MCAST_NEXT_DROP,
1213     LOOKUP_IP_DST_MCAST_NEXT_RPF,
1214     LOOKUP_IP_DST_MCAST_N_NEXT,
1215 } mfib_forward_lookup_next_t;
1216
1217 always_inline uword
1218 lookup_dpo_ip_dst_mcast_inline (vlib_main_t * vm,
1219                                 vlib_node_runtime_t * node,
1220                                 vlib_frame_t * from_frame,
1221                                 int is_v4)
1222 {
1223     u32 n_left_from, next_index, * from, * to_next;
1224
1225     from = vlib_frame_vector_args (from_frame);
1226     n_left_from = from_frame->n_vectors;
1227
1228     next_index = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1229
1230     while (n_left_from > 0)
1231     {
1232         u32 n_left_to_next;
1233
1234         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
1235
1236         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1237         /*   } */
1238
1239         while (n_left_from > 0 && n_left_to_next > 0)
1240         {
1241             u32 bi0, lkdi0, fib_index0,  next0;
1242             const lookup_dpo_t * lkd0;
1243             fib_node_index_t mfei0;
1244             vlib_buffer_t * b0;
1245
1246             bi0 = from[0];
1247             to_next[0] = bi0;
1248             from += 1;
1249             to_next += 1;
1250             n_left_from -= 1;
1251             n_left_to_next -= 1;
1252
1253             b0 = vlib_get_buffer (vm, bi0);
1254
1255             /* dst lookup was done by mpls lookup */
1256             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1257             lkd0 = lookup_dpo_get(lkdi0);
1258             fib_index0 = lkd0->lkd_fib_index;
1259             next0 = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1260
1261             if (is_v4)
1262             {
1263                 ip4_header_t * ip0;
1264
1265                 ip0 = vlib_buffer_get_current (b0);
1266                 mfei0 = ip4_mfib_table_lookup(ip4_mfib_get(fib_index0),
1267                                               &ip0->src_address,
1268                                               &ip0->dst_address,
1269                                               64);
1270                 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1271                 {
1272                     lookup_trace_t *tr = vlib_add_trace (vm, node,
1273                                                          b0, sizeof (*tr));
1274                     tr->fib_index = fib_index0;
1275                     tr->lbi = mfei0;
1276                     tr->addr.ip4 = ip0->dst_address;
1277                 }
1278             }
1279             else
1280             {
1281                 ip6_header_t * ip0;
1282
1283                 ip0 = vlib_buffer_get_current (b0);
1284                 mfei0 = ip6_mfib_table_fwd_lookup(ip6_mfib_get(fib_index0),
1285                                                   &ip0->src_address,
1286                                                   &ip0->dst_address);
1287                 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1288                 {
1289                     lookup_trace_t *tr = vlib_add_trace (vm, node,
1290                                                          b0, sizeof (*tr));
1291                     tr->fib_index = fib_index0;
1292                     tr->lbi = mfei0;
1293                     tr->addr.ip6 = ip0->dst_address;
1294                 }
1295             }
1296
1297             vnet_buffer (b0)->ip.adj_index[VLIB_TX] = mfei0;
1298
1299             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
1300                 vnet_buffer2(b0)->loop_counter = 0;
1301                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
1302             }
1303
1304             vnet_buffer2(b0)->loop_counter++;
1305
1306             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
1307                 next0 = LOOKUP_IP_DST_MCAST_NEXT_DROP;
1308
1309             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1310                                             n_left_to_next, bi0, next0);
1311         }
1312         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1313     }
1314     return from_frame->n_vectors;
1315 }
1316
1317 VLIB_NODE_FN (lookup_ip4_dst_mcast_node) (vlib_main_t * vm,
1318                       vlib_node_runtime_t * node,
1319                       vlib_frame_t * from_frame)
1320 {
1321     return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 1));
1322 }
1323
1324 VLIB_REGISTER_NODE (lookup_ip4_dst_mcast_node) = {
1325     .name = "lookup-ip4-dst-mcast",
1326     .vector_size = sizeof (u32),
1327
1328     .format_trace = format_lookup_trace,
1329     .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1330     .next_nodes = {
1331         [LOOKUP_IP_DST_MCAST_NEXT_DROP] = "ip4-drop",
1332         [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip4-mfib-forward-rpf",
1333     },
1334 };
1335
1336 VLIB_NODE_FN (lookup_ip6_dst_mcast_node) (vlib_main_t * vm,
1337                       vlib_node_runtime_t * node,
1338                       vlib_frame_t * from_frame)
1339 {
1340     return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 0));
1341 }
1342
1343 VLIB_REGISTER_NODE (lookup_ip6_dst_mcast_node) = {
1344     .name = "lookup-ip6-dst-mcast",
1345     .vector_size = sizeof (u32),
1346
1347     .format_trace = format_lookup_trace,
1348     .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1349     .next_nodes = {
1350         [LOOKUP_IP_DST_MCAST_NEXT_DROP] = "ip6-drop",
1351         [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip6-mfib-forward-rpf",
1352     },
1353 };
1354
1355 static void
1356 lookup_dpo_mem_show (void)
1357 {
1358     fib_show_memory_usage("Lookup",
1359                           pool_elts(lookup_dpo_pool),
1360                           pool_len(lookup_dpo_pool),
1361                           sizeof(lookup_dpo_t));
1362 }
1363
1364 const static dpo_vft_t lkd_vft = {
1365     .dv_lock = lookup_dpo_lock,
1366     .dv_unlock = lookup_dpo_unlock,
1367     .dv_format = format_lookup_dpo,
1368 };
1369 const static dpo_vft_t lkd_vft_w_mem_show = {
1370     .dv_lock = lookup_dpo_lock,
1371     .dv_unlock = lookup_dpo_unlock,
1372     .dv_format = format_lookup_dpo,
1373     .dv_mem_show = lookup_dpo_mem_show,
1374 };
1375
1376 const static char* const lookup_src_ip4_nodes[] =
1377 {
1378     "lookup-ip4-src",
1379     NULL,
1380 };
1381 const static char* const lookup_src_ip6_nodes[] =
1382 {
1383     "lookup-ip6-src",
1384     NULL,
1385 };
1386 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
1387 {
1388     [DPO_PROTO_IP4]  = lookup_src_ip4_nodes,
1389     [DPO_PROTO_IP6]  = lookup_src_ip6_nodes,
1390     [DPO_PROTO_MPLS] = NULL,
1391 };
1392
1393 const static char* const lookup_dst_ip4_nodes[] =
1394 {
1395     "lookup-ip4-dst",
1396     NULL,
1397 };
1398 const static char* const lookup_dst_ip6_nodes[] =
1399 {
1400     "lookup-ip6-dst",
1401     NULL,
1402 };
1403 const static char* const lookup_dst_mpls_nodes[] =
1404 {
1405     "lookup-mpls-dst",
1406     NULL,
1407 };
1408 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
1409 {
1410     [DPO_PROTO_IP4]  = lookup_dst_ip4_nodes,
1411     [DPO_PROTO_IP6]  = lookup_dst_ip6_nodes,
1412     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
1413 };
1414
1415 const static char* const lookup_dst_mcast_ip4_nodes[] =
1416 {
1417     "lookup-ip4-dst-mcast",
1418     NULL,
1419 };
1420 const static char* const lookup_dst_mcast_ip6_nodes[] =
1421 {
1422     "lookup-ip6-dst-mcast",
1423     NULL,
1424 };
1425 const static char* const * const lookup_dst_mcast_nodes[DPO_PROTO_NUM] =
1426 {
1427     [DPO_PROTO_IP4]  = lookup_dst_mcast_ip4_nodes,
1428     [DPO_PROTO_IP6]  = lookup_dst_mcast_ip6_nodes,
1429 };
1430
1431 const static char* const lookup_dst_from_interface_ip4_nodes[] =
1432 {
1433     "lookup-ip4-dst-itf",
1434     NULL,
1435 };
1436 const static char* const lookup_dst_from_interface_ip6_nodes[] =
1437 {
1438     "lookup-ip6-dst-itf",
1439     NULL,
1440 };
1441 const static char* const lookup_dst_from_interface_mpls_nodes[] =
1442 {
1443     "lookup-mpls-dst-itf",
1444     NULL,
1445 };
1446 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
1447 {
1448     [DPO_PROTO_IP4]  = lookup_dst_from_interface_ip4_nodes,
1449     [DPO_PROTO_IP6]  = lookup_dst_from_interface_ip6_nodes,
1450     [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
1451 };
1452
1453 static clib_error_t *
1454 lookup_dpo_show (vlib_main_t * vm,
1455                  unformat_input_t * input,
1456                  vlib_cli_command_t * cmd)
1457 {
1458     index_t lkdi = INDEX_INVALID;
1459
1460     while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1461     {
1462         if (unformat (input, "%d", &lkdi))
1463             ;
1464         else
1465             break;
1466     }
1467
1468     if (INDEX_INVALID != lkdi)
1469     {
1470         if (pool_is_free_index(lookup_dpo_pool, lkdi))
1471                 vlib_cli_output (vm, "no such index %d", lkdi);
1472         else
1473                 vlib_cli_output (vm, "%U", format_lookup_dpo, lkdi);
1474     }
1475     else
1476     {
1477         lookup_dpo_t *lkd;
1478
1479         pool_foreach(lkd, lookup_dpo_pool,
1480         ({
1481             vlib_cli_output (vm, "[@%d] %U",
1482                              lookup_dpo_get_index(lkd),
1483                              format_lookup_dpo,
1484                              lookup_dpo_get_index(lkd));
1485         }));
1486     }
1487
1488     return 0;
1489 }
1490
1491 VLIB_CLI_COMMAND (replicate_show_command, static) = {
1492     .path = "show lookup-dpo",
1493     .short_help = "show lookup-dpo [<index>]",
1494     .function = lookup_dpo_show,
1495 };
1496
1497 void
1498 lookup_dpo_module_init (void)
1499 {
1500     dpo_register(DPO_LOOKUP, &lkd_vft_w_mem_show, NULL);
1501
1502     /*
1503      * There are various sorts of lookup; src or dst addr v4 /v6 etc.
1504      * there isn't an object type for each (there is only the lookup_dpo_t),
1505      * but, for performance reasons, there is a data plane function, and hence
1506      * VLIB node for each. VLIB graph node construction is based on DPO types
1507      * so we create sub-types.
1508      */
1509     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
1510         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
1511     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
1512         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
1513     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST] =
1514         dpo_register_new_type(&lkd_vft, lookup_dst_mcast_nodes);
1515     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
1516         dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes);
1517 }