src/vnet/dpo/lookup_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/lookup_dpo.h>
  18 #include <vnet/dpo/load_balance_map.h>
  19 #include <vnet/mpls/mpls_lookup.h>
  20 #include <vnet/fib/fib_table.h>
  21 #include <vnet/fib/ip4_fib.h>
  22 #include <vnet/fib/ip6_fib.h>
  23 #include <vnet/fib/mpls_fib.h>
  24 #include <vnet/mfib/mfib_table.h>
  25 #include <vnet/mfib/ip4_mfib.h>
  26 #include <vnet/mfib/ip6_mfib.h>
  27
  28 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
  29 static const char *const lookup_cast_names[] = LOOKUP_CASTS;
  30
  31 /**
  32  * If a packet encounters a lookup DPO more than the many times
  33  * then we assume there is a loop in the forward graph and drop the packet
  34  */
  35 #define MAX_LUKPS_PER_PACKET 4
  36
  37 /**
  38  * @brief Enumeration of the lookup subtypes
  39  */
  40 typedef enum lookup_sub_type_t_
  41 {
  42     LOOKUP_SUB_TYPE_SRC,
  43     LOOKUP_SUB_TYPE_DST,
  44     LOOKUP_SUB_TYPE_DST_MCAST,
  45     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
  46 } lookup_sub_type_t;
  47 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
  48
  49 #define FOR_EACH_LOOKUP_SUB_TYPE(_st)                                   \
  50     for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
  51
  52 /**
  53  * @brief pool of all MPLS Label DPOs
  54  */
  55 lookup_dpo_t *lookup_dpo_pool;
  56
  57 /**
  58  * @brief An array of registered DPO type values for the sub-types
  59  */
  60 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
  61
  62 static lookup_dpo_t *
  63 lookup_dpo_alloc (void)
  64 {
  65     lookup_dpo_t *lkd;
  66     vlib_main_t *vm;
  67     u8 did_barrier_sync;
  68
  69     dpo_pool_barrier_sync (vm, lookup_dpo_pool, did_barrier_sync);
  70     pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
  71     dpo_pool_barrier_release (vm, did_barrier_sync);
  72
  73     return (lkd);
  74 }
  75
  76 static index_t
  77 lookup_dpo_get_index (lookup_dpo_t *lkd)
  78 {
  79     return (lkd - lookup_dpo_pool);
  80 }
  81
  82 static void
  83 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
  84                           dpo_proto_t proto,
  85                           lookup_cast_t cast,
  86                           lookup_input_t input,
  87                           lookup_table_t table_config,
  88                           dpo_id_t *dpo)
  89 {
  90     lookup_dpo_t *lkd;
  91     dpo_type_t type;
  92
  93     lkd = lookup_dpo_alloc();
  94     lkd->lkd_fib_index = fib_index;
  95     lkd->lkd_proto = proto;
  96     lkd->lkd_input = input;
  97     lkd->lkd_table = table_config;
  98     lkd->lkd_cast  = cast;
  99
 100     /*
 101      * use the input type to select the lookup sub-type
 102      */
 103     type = 0;
 104
 105     switch (input)
 106     {
 107     case LOOKUP_INPUT_SRC_ADDR:
 108         type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
 109         break;
 110     case LOOKUP_INPUT_DST_ADDR:
 111         switch (table_config)
 112         {
 113         case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
 114             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
 115             break;
 116         case LOOKUP_TABLE_FROM_CONFIG:
 117             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
 118             break;
 119         }
 120         if (LOOKUP_MULTICAST == cast)
 121         {
 122             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST];
 123         }
 124     }
 125
 126     if (0 == type)
 127     {
 128         dpo_reset(dpo);
 129     }
 130     else
 131     {
 132         dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
 133     }
 134 }
 135
 136 void
 137 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
 138                                     dpo_proto_t proto,
 139                                     lookup_cast_t cast,
 140                                     lookup_input_t input,
 141                                     lookup_table_t table_config,
 142                                     dpo_id_t *dpo)
 143 {
 144     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 145     {
 146         if (LOOKUP_UNICAST == cast)
 147         {
 148             fib_table_lock(fib_index,
 149                            dpo_proto_to_fib(proto),
 150                            FIB_SOURCE_RR);
 151         }
 152         else
 153         {
 154             mfib_table_lock(fib_index,
 155                             dpo_proto_to_fib(proto),
 156                             MFIB_SOURCE_RR);
 157         }
 158     }
 159     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 160 }
 161
 162 void
 163 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 164                                    dpo_proto_t proto,
 165                                    lookup_cast_t cast,
 166                                    lookup_input_t input,
 167                                    lookup_table_t table_config,
 168                                    dpo_id_t *dpo)
 169 {
 170     fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
 171
 172     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 173     {
 174         if (LOOKUP_UNICAST == cast)
 175         {
 176             fib_index =
 177                 fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 178                                                   table_id,
 179                                                   FIB_SOURCE_RR);
 180         }
 181         else
 182         {
 183             fib_index =
 184                 mfib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 185                                                    table_id,
 186                                                    MFIB_SOURCE_RR);
 187         }
 188     }
 189
 190     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
 191     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 192 }
 193
 194 u8*
 195 format_lookup_dpo (u8 *s, va_list *args)
 196 {
 197     index_t index = va_arg (*args, index_t);
 198     lookup_dpo_t *lkd;
 199
 200     lkd = lookup_dpo_get(index);
 201
 202     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
 203     {
 204         s = format(s, "%s,%s lookup in interface's %U table",
 205                    lookup_input_names[lkd->lkd_input],
 206                    lookup_cast_names[lkd->lkd_cast],
 207                    format_dpo_proto, lkd->lkd_proto);
 208     }
 209     else
 210     {
 211         if (LOOKUP_UNICAST == lkd->lkd_cast)
 212         {
 213             s = format(s, "%s,%s lookup in %U",
 214                        lookup_input_names[lkd->lkd_input],
 215                        lookup_cast_names[lkd->lkd_cast],
 216                        format_fib_table_name, lkd->lkd_fib_index,
 217                        dpo_proto_to_fib(lkd->lkd_proto));
 218         }
 219         else
 220         {
 221             s = format(s, "%s,%s lookup in %U",
 222                        lookup_input_names[lkd->lkd_input],
 223                        lookup_cast_names[lkd->lkd_cast],
 224                        format_mfib_table_name, lkd->lkd_fib_index,
 225                        dpo_proto_to_fib(lkd->lkd_proto));
 226         }
 227     }
 228     return (s);
 229 }
 230
 231 static void
 232 lookup_dpo_lock (dpo_id_t *dpo)
 233 {
 234     lookup_dpo_t *lkd;
 235
 236     lkd = lookup_dpo_get(dpo->dpoi_index);
 237
 238     lkd->lkd_locks++;
 239 }
 240
 241 static void
 242 lookup_dpo_unlock (dpo_id_t *dpo)
 243 {
 244     lookup_dpo_t *lkd;
 245
 246     lkd = lookup_dpo_get(dpo->dpoi_index);
 247
 248     lkd->lkd_locks--;
 249
 250     if (0 == lkd->lkd_locks)
 251     {
 252         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
 253         {
 254             if (LOOKUP_UNICAST == lkd->lkd_cast)
 255             {
 256                 fib_table_unlock(lkd->lkd_fib_index,
 257                                  dpo_proto_to_fib(lkd->lkd_proto),
 258                                  FIB_SOURCE_RR);
 259             }
 260             else
 261             {
 262                 mfib_table_unlock(lkd->lkd_fib_index,
 263                                   dpo_proto_to_fib(lkd->lkd_proto),
 264                                   MFIB_SOURCE_RR);
 265             }
 266         }
 267         pool_put(lookup_dpo_pool, lkd);
 268     }
 269 }
 270
 271 always_inline void
 272 ip4_src_fib_lookup_one (u32 src_fib_index0,
 273                         const ip4_address_t * addr0,
 274                         u32 * src_adj_index0)
 275 {
 276     ip4_fib_mtrie_leaf_t leaf0;
 277     ip4_fib_mtrie_t * mtrie0;
 278
 279     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 280
 281     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 282     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 283     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 284
 285     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 286 }
 287
 288 always_inline void
 289 ip4_src_fib_lookup_two (u32 src_fib_index0,
 290                         u32 src_fib_index1,
 291                         const ip4_address_t * addr0,
 292                         const ip4_address_t * addr1,
 293                         u32 * src_adj_index0,
 294                         u32 * src_adj_index1)
 295 {
 296     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 297     ip4_fib_mtrie_t * mtrie0, * mtrie1;
 298
 299     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 300     mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
 301
 302     leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
 303     leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, addr1);
 304
 305     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 306     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
 307
 308     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 309     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
 310
 311     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 312     src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 313 }
 314
 315 /**
 316  * @brief Lookup trace  data
 317  */
 318 typedef struct lookup_trace_t_
 319 {
 320     union {
 321         ip46_address_t addr;
 322         mpls_unicast_header_t hdr;
 323     };
 324     fib_node_index_t fib_index;
 325     index_t lbi;
 326 } lookup_trace_t;
 327
 328
 329 always_inline uword
 330 lookup_dpo_ip4_inline (vlib_main_t * vm,
 331                        vlib_node_runtime_t * node,
 332                        vlib_frame_t * from_frame,
 333                        int input_src_addr,
 334                        int table_from_interface)
 335 {
 336     u32 n_left_from, next_index, * from, * to_next;
 337     u32 thread_index = vlib_get_thread_index();
 338     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 339
 340     from = vlib_frame_vector_args (from_frame);
 341     n_left_from = from_frame->n_vectors;
 342
 343     next_index = node->cached_next_index;
 344
 345     while (n_left_from > 0)
 346     {
 347         u32 n_left_to_next;
 348
 349         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 350
 351         while (n_left_from >= 4 && n_left_to_next > 2)
 352         {
 353             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 354             flow_hash_config_t flow_hash_config0;
 355             const ip4_address_t *input_addr0;
 356             const load_balance_t *lb0;
 357             const lookup_dpo_t * lkd0;
 358             const ip4_header_t * ip0;
 359             const dpo_id_t *dpo0;
 360             vlib_buffer_t * b0;
 361             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 362             flow_hash_config_t flow_hash_config1;
 363             const ip4_address_t *input_addr1;
 364             const load_balance_t *lb1;
 365             const lookup_dpo_t * lkd1;
 366             const ip4_header_t * ip1;
 367             const dpo_id_t *dpo1;
 368             vlib_buffer_t * b1;
 369
 370             /* Prefetch next iteration. */
 371             {
 372                 vlib_buffer_t * p2, * p3;
 373
 374                 p2 = vlib_get_buffer (vm, from[2]);
 375                 p3 = vlib_get_buffer (vm, from[3]);
 376
 377                 vlib_prefetch_buffer_header (p2, LOAD);
 378                 vlib_prefetch_buffer_header (p3, LOAD);
 379
 380                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 381                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 382             }
 383
 384             bi0 = from[0];
 385             to_next[0] = bi0;
 386             bi1 = from[1];
 387             to_next[1] = bi1;
 388             from += 2;
 389             to_next += 2;
 390             n_left_from -= 2;
 391             n_left_to_next -= 2;
 392
 393             b0 = vlib_get_buffer (vm, bi0);
 394             ip0 = vlib_buffer_get_current (b0);
 395             b1 = vlib_get_buffer (vm, bi1);
 396             ip1 = vlib_buffer_get_current (b1);
 397
 398             /* dst lookup was done by ip4 lookup */
 399             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 400             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 401             lkd0 = lookup_dpo_get(lkdi0);
 402             lkd1 = lookup_dpo_get(lkdi1);
 403
 404             /*
 405              * choose between a lookup using the fib index in the DPO
 406              * or getting the FIB index from the interface.
 407              */
 408             if (table_from_interface)
 409             {
 410                 fib_index0 =
 411                     ip4_fib_table_get_index_for_sw_if_index(
 412                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 413                 fib_index1 =
 414                     ip4_fib_table_get_index_for_sw_if_index(
 415                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 416             }
 417             else
 418             {
 419                 fib_index0 = lkd0->lkd_fib_index;
 420                 fib_index1 = lkd1->lkd_fib_index;
 421             }
 422
 423             /*
 424              * choose between a source or destination address lookup in the table
 425              */
 426             if (input_src_addr)
 427             {
 428                 input_addr0 = &ip0->src_address;
 429                 input_addr1 = &ip1->src_address;
 430             }
 431             else
 432             {
 433                 input_addr0 = &ip0->dst_address;
 434                 input_addr1 = &ip1->dst_address;
 435             }
 436
 437             /* do lookup */
 438             ip4_src_fib_lookup_two (fib_index0, fib_index1,
 439                                     input_addr0, input_addr1,
 440                                     &lbi0, &lbi1);
 441             lb0 = load_balance_get(lbi0);
 442             lb1 = load_balance_get(lbi1);
 443
 444             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 445             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 446
 447             /* Use flow hash to compute multipath adjacency. */
 448             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 449             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 450
 451             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 452             {
 453                 flow_hash_config0 = lb0->lb_hash_config;
 454                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 455                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 456             }
 457
 458             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 459             {
 460                 flow_hash_config1 = lb1->lb_hash_config;
 461                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 462                     ip4_compute_flow_hash (ip1, flow_hash_config1);
 463             }
 464
 465             dpo0 = load_balance_get_bucket_i(lb0,
 466                                              (hash_c0 &
 467                                               (lb0->lb_n_buckets_minus_1)));
 468             dpo1 = load_balance_get_bucket_i(lb1,
 469                                              (hash_c1 &
 470                                               (lb1->lb_n_buckets_minus_1)));
 471
 472             next0 = dpo0->dpoi_next_node;
 473             next1 = dpo1->dpoi_next_node;
 474             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 475             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 476
 477             vlib_increment_combined_counter
 478                 (cm, thread_index, lbi0, 1,
 479                  vlib_buffer_length_in_chain (vm, b0));
 480             vlib_increment_combined_counter
 481                 (cm, thread_index, lbi1, 1,
 482                  vlib_buffer_length_in_chain (vm, b1));
 483
 484             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 485                 vnet_buffer2(b0)->loop_counter = 0;
 486                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 487             }
 488             if (!(b1->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 489                 vnet_buffer2(b1)->loop_counter = 0;
 490                 b1->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 491             }
 492
 493             vnet_buffer2(b0)->loop_counter++;
 494             vnet_buffer2(b1)->loop_counter++;
 495
 496             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 497                 next0 = IP_LOOKUP_NEXT_DROP;
 498             if (PREDICT_FALSE(vnet_buffer2(b1)->loop_counter > MAX_LUKPS_PER_PACKET))
 499                 next1 = IP_LOOKUP_NEXT_DROP;
 500
 501             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 502             {
 503                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 504                                                      b0, sizeof (*tr));
 505                 tr->fib_index = fib_index0;
 506                 tr->lbi = lbi0;
 507                 tr->addr.ip4 = *input_addr0;
 508             }
 509             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 510             {
 511                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 512                                                      b1, sizeof (*tr));
 513                 tr->fib_index = fib_index1;
 514                 tr->lbi = lbi1;
 515                 tr->addr.ip4 = *input_addr1;
 516             }
 517
 518             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 519                                              to_next, n_left_to_next,
 520                                              bi0, bi1, next0, next1);
 521         }
 522
 523         while (n_left_from > 0 && n_left_to_next > 0)
 524         {
 525             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 526             flow_hash_config_t flow_hash_config0;
 527             const ip4_address_t *input_addr;
 528             const load_balance_t *lb0;
 529             const lookup_dpo_t * lkd0;
 530             const ip4_header_t * ip0;
 531             const dpo_id_t *dpo0;
 532             vlib_buffer_t * b0;
 533
 534             bi0 = from[0];
 535             to_next[0] = bi0;
 536             from += 1;
 537             to_next += 1;
 538             n_left_from -= 1;
 539             n_left_to_next -= 1;
 540
 541             b0 = vlib_get_buffer (vm, bi0);
 542             ip0 = vlib_buffer_get_current (b0);
 543
 544             /* dst lookup was done by ip4 lookup */
 545             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 546             lkd0 = lookup_dpo_get(lkdi0);
 547
 548             /*
 549              * choose between a lookup using the fib index in the DPO
 550              * or getting the FIB index from the interface.
 551              */
 552             if (table_from_interface)
 553             {
 554                 fib_index0 =
 555                     ip4_fib_table_get_index_for_sw_if_index(
 556                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 557             }
 558             else
 559             {
 560                 fib_index0 = lkd0->lkd_fib_index;
 561             }
 562
 563             /*
 564              * choose between a source or destination address lookup in the table
 565              */
 566             if (input_src_addr)
 567             {
 568                 input_addr = &ip0->src_address;
 569             }
 570             else
 571             {
 572                 input_addr = &ip0->dst_address;
 573             }
 574
 575             /* do lookup */
 576             ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
 577             lb0 = load_balance_get(lbi0);
 578
 579             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 580
 581             /* Use flow hash to compute multipath adjacency. */
 582             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 583
 584             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 585             {
 586                 flow_hash_config0 = lb0->lb_hash_config;
 587                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 588                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 589             }
 590
 591             dpo0 = load_balance_get_bucket_i(lb0,
 592                                              (hash_c0 &
 593                                               (lb0->lb_n_buckets_minus_1)));
 594
 595             next0 = dpo0->dpoi_next_node;
 596             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 597
 598             vlib_increment_combined_counter
 599                 (cm, thread_index, lbi0, 1,
 600                  vlib_buffer_length_in_chain (vm, b0));
 601
 602             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 603                 vnet_buffer2(b0)->loop_counter = 0;
 604                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 605             }
 606
 607             vnet_buffer2(b0)->loop_counter++;
 608
 609             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 610                 next0 = IP_LOOKUP_NEXT_DROP;
 611
 612             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 613             {
 614                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 615                                                      b0, sizeof (*tr));
 616                 tr->fib_index = fib_index0;
 617                 tr->lbi = lbi0;
 618                 tr->addr.ip4 = *input_addr;
 619             }
 620
 621             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 622                                             n_left_to_next, bi0, next0);
 623         }
 624         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 625     }
 626     return from_frame->n_vectors;
 627 }
 628
 629 static u8 *
 630 format_lookup_trace (u8 * s, va_list * args)
 631 {
 632     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 633     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 634     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 635     u32 indent = format_get_indent (s);
 636     s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
 637                 format_white_space, indent,
 638                 t->fib_index,
 639                 format_ip46_address, &t->addr, IP46_TYPE_ANY,
 640                 t->lbi);
 641     return s;
 642 }
 643
 644 VLIB_NODE_FN (lookup_ip4_dst_node) (vlib_main_t * vm,
 645                 vlib_node_runtime_t * node,
 646                 vlib_frame_t * from_frame)
 647 {
 648     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
 649 }
 650
 651 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
 652     .name = "lookup-ip4-dst",
 653     .vector_size = sizeof (u32),
 654     .sibling_of = "ip4-lookup",
 655     .format_trace = format_lookup_trace,
 656 };
 657
 658 VLIB_NODE_FN (lookup_ip4_dst_itf_node) (vlib_main_t * vm,
 659                     vlib_node_runtime_t * node,
 660                     vlib_frame_t * from_frame)
 661 {
 662     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
 663 }
 664
 665 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
 666     .name = "lookup-ip4-dst-itf",
 667     .vector_size = sizeof (u32),
 668     .sibling_of = "ip4-lookup",
 669     .format_trace = format_lookup_trace,
 670 };
 671
 672 VLIB_NODE_FN (lookup_ip4_src_node) (vlib_main_t * vm,
 673                 vlib_node_runtime_t * node,
 674                 vlib_frame_t * from_frame)
 675 {
 676     return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
 677 }
 678
 679 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
 680     .name = "lookup-ip4-src",
 681     .vector_size = sizeof (u32),
 682     .format_trace = format_lookup_trace,
 683     .sibling_of = "ip4-lookup",
 684 };
 685
 686 always_inline uword
 687 lookup_dpo_ip6_inline (vlib_main_t * vm,
 688                        vlib_node_runtime_t * node,
 689                        vlib_frame_t * from_frame,
 690                        int input_src_addr,
 691                        int table_from_interface)
 692 {
 693     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 694     u32 n_left_from, next_index, * from, * to_next;
 695     u32 thread_index = vlib_get_thread_index();
 696
 697     from = vlib_frame_vector_args (from_frame);
 698     n_left_from = from_frame->n_vectors;
 699
 700     next_index = node->cached_next_index;
 701
 702     while (n_left_from > 0)
 703     {
 704         u32 n_left_to_next;
 705
 706         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 707
 708         while (n_left_from >= 4 && n_left_to_next > 2)
 709         {
 710             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 711             flow_hash_config_t flow_hash_config0;
 712             const ip6_address_t *input_addr0;
 713             const load_balance_t *lb0;
 714             const lookup_dpo_t * lkd0;
 715             const ip6_header_t * ip0;
 716             const dpo_id_t *dpo0;
 717             vlib_buffer_t * b0;
 718             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 719             flow_hash_config_t flow_hash_config1;
 720             const ip6_address_t *input_addr1;
 721             const load_balance_t *lb1;
 722             const lookup_dpo_t * lkd1;
 723             const ip6_header_t * ip1;
 724             const dpo_id_t *dpo1;
 725             vlib_buffer_t * b1;
 726
 727             /* Prefetch next iteration. */
 728             {
 729                 vlib_buffer_t * p2, * p3;
 730
 731                 p2 = vlib_get_buffer (vm, from[2]);
 732                 p3 = vlib_get_buffer (vm, from[3]);
 733
 734                 vlib_prefetch_buffer_header (p2, LOAD);
 735                 vlib_prefetch_buffer_header (p3, LOAD);
 736
 737                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 738                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 739             }
 740
 741             bi0 = from[0];
 742             to_next[0] = bi0;
 743             bi1 = from[1];
 744             to_next[1] = bi1;
 745             from += 2;
 746             to_next += 2;
 747             n_left_from -= 2;
 748             n_left_to_next -= 2;
 749
 750             b0 = vlib_get_buffer (vm, bi0);
 751             ip0 = vlib_buffer_get_current (b0);
 752             b1 = vlib_get_buffer (vm, bi1);
 753             ip1 = vlib_buffer_get_current (b1);
 754
 755             /* dst lookup was done by ip6 lookup */
 756             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 757             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 758             lkd0 = lookup_dpo_get(lkdi0);
 759             lkd1 = lookup_dpo_get(lkdi1);
 760
 761             /*
 762              * choose between a lookup using the fib index in the DPO
 763              * or getting the FIB index from the interface.
 764              */
 765             if (table_from_interface)
 766             {
 767                 fib_index0 =
 768                     ip6_fib_table_get_index_for_sw_if_index(
 769                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 770                 fib_index1 =
 771                     ip6_fib_table_get_index_for_sw_if_index(
 772                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 773             }
 774             else
 775             {
 776                 fib_index0 = lkd0->lkd_fib_index;
 777                 fib_index1 = lkd1->lkd_fib_index;
 778             }
 779
 780             /*
 781              * choose between a source or destination address lookup in the table
 782              */
 783             if (input_src_addr)
 784             {
 785                 input_addr0 = &ip0->src_address;
 786                 input_addr1 = &ip1->src_address;
 787             }
 788             else
 789             {
 790                 input_addr0 = &ip0->dst_address;
 791                 input_addr1 = &ip1->dst_address;
 792             }
 793
 794             /* do src lookup */
 795             lbi0 = ip6_fib_table_fwding_lookup(
 796                                                fib_index0,
 797                                                input_addr0);
 798             lbi1 = ip6_fib_table_fwding_lookup(
 799                                                fib_index1,
 800                                                input_addr1);
 801             lb0 = load_balance_get(lbi0);
 802             lb1 = load_balance_get(lbi1);
 803
 804             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 805             vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
 806
 807             /* Use flow hash to compute multipath adjacency. */
 808             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 809             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 810
 811             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 812                 vnet_buffer2(b0)->loop_counter = 0;
 813                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 814             }
 815             if (!(b1->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 816                 vnet_buffer2(b1)->loop_counter = 0;
 817                 b1->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 818             }
 819
 820             vnet_buffer2(b0)->loop_counter++;
 821             vnet_buffer2(b1)->loop_counter++;
 822
 823             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 824                 next0 = IP_LOOKUP_NEXT_DROP;
 825             if (PREDICT_FALSE(vnet_buffer2(b1)->loop_counter > MAX_LUKPS_PER_PACKET))
 826                 next1 = IP_LOOKUP_NEXT_DROP;
 827
 828             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 829             {
 830                 flow_hash_config0 = lb0->lb_hash_config;
 831                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 832                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 833             }
 834
 835             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 836             {
 837                 flow_hash_config1 = lb1->lb_hash_config;
 838                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 839                     ip6_compute_flow_hash (ip1, flow_hash_config1);
 840             }
 841
 842             dpo0 = load_balance_get_bucket_i(lb0,
 843                                              (hash_c0 &
 844                                               (lb0->lb_n_buckets_minus_1)));
 845             dpo1 = load_balance_get_bucket_i(lb1,
 846                                              (hash_c1 &
 847                                               (lb1->lb_n_buckets_minus_1)));
 848
 849             next0 = dpo0->dpoi_next_node;
 850             next1 = dpo1->dpoi_next_node;
 851             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 852             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 853
 854             vlib_increment_combined_counter
 855                 (cm, thread_index, lbi0, 1,
 856                  vlib_buffer_length_in_chain (vm, b0));
 857             vlib_increment_combined_counter
 858                 (cm, thread_index, lbi1, 1,
 859                  vlib_buffer_length_in_chain (vm, b1));
 860
 861             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 862             {
 863                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 864                                                      b0, sizeof (*tr));
 865                 tr->fib_index = fib_index0;
 866                 tr->lbi = lbi0;
 867                 tr->addr.ip6 = *input_addr0;
 868             }
 869             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 870             {
 871                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 872                                                      b1, sizeof (*tr));
 873                 tr->fib_index = fib_index1;
 874                 tr->lbi = lbi1;
 875                 tr->addr.ip6 = *input_addr1;
 876             }
 877             vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
 878                                             n_left_to_next, bi0, bi1,
 879                                             next0, next1);
 880         }
 881         while (n_left_from > 0 && n_left_to_next > 0)
 882         {
 883             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 884             flow_hash_config_t flow_hash_config0;
 885             const ip6_address_t *input_addr0;
 886             const load_balance_t *lb0;
 887             const lookup_dpo_t * lkd0;
 888             const ip6_header_t * ip0;
 889             const dpo_id_t *dpo0;
 890             vlib_buffer_t * b0;
 891
 892             bi0 = from[0];
 893             to_next[0] = bi0;
 894             from += 1;
 895             to_next += 1;
 896             n_left_from -= 1;
 897             n_left_to_next -= 1;
 898
 899             b0 = vlib_get_buffer (vm, bi0);
 900             ip0 = vlib_buffer_get_current (b0);
 901
 902             /* dst lookup was done by ip6 lookup */
 903             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 904             lkd0 = lookup_dpo_get(lkdi0);
 905
 906             /*
 907              * choose between a lookup using the fib index in the DPO
 908              * or getting the FIB index from the interface.
 909              */
 910             if (table_from_interface)
 911             {
 912                 fib_index0 =
 913                     ip6_fib_table_get_index_for_sw_if_index(
 914                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 915             }
 916             else
 917             {
 918                 fib_index0 = lkd0->lkd_fib_index;
 919             }
 920
 921             /*
 922              * choose between a source or destination address lookup in the table
 923              */
 924             if (input_src_addr)
 925             {
 926                 input_addr0 = &ip0->src_address;
 927             }
 928             else
 929             {
 930                 input_addr0 = &ip0->dst_address;
 931             }
 932
 933             /* do src lookup */
 934             lbi0 = ip6_fib_table_fwding_lookup(
 935                                                fib_index0,
 936                                                input_addr0);
 937             lb0 = load_balance_get(lbi0);
 938
 939             vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
 940
 941             /* Use flow hash to compute multipath adjacency. */
 942             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 943
 944             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 945             {
 946                 flow_hash_config0 = lb0->lb_hash_config;
 947                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 948                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 949             }
 950
 951             dpo0 = load_balance_get_bucket_i(lb0,
 952                                              (hash_c0 &
 953                                               (lb0->lb_n_buckets_minus_1)));
 954
 955             next0 = dpo0->dpoi_next_node;
 956             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 957
 958             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
 959                 vnet_buffer2(b0)->loop_counter = 0;
 960                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
 961             }
 962
 963             vnet_buffer2(b0)->loop_counter++;
 964
 965             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
 966                 next0 = IP_LOOKUP_NEXT_DROP;
 967
 968             vlib_increment_combined_counter
 969                 (cm, thread_index, lbi0, 1,
 970                  vlib_buffer_length_in_chain (vm, b0));
 971
 972             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 973             {
 974                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 975                                                      b0, sizeof (*tr));
 976                 tr->fib_index = fib_index0;
 977                 tr->lbi = lbi0;
 978                 tr->addr.ip6 = *input_addr0;
 979             }
 980             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 981                                             n_left_to_next, bi0, next0);
 982         }
 983         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 984     }
 985     return from_frame->n_vectors;
 986 }
 987
 988 VLIB_NODE_FN (lookup_ip6_dst_node) (vlib_main_t * vm,
 989                 vlib_node_runtime_t * node,
 990                 vlib_frame_t * from_frame)
 991 {
 992     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 0));
 993 }
 994
 995 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
 996     .name = "lookup-ip6-dst",
 997     .vector_size = sizeof (u32),
 998     .format_trace = format_lookup_trace,
 999     .sibling_of = "ip6-lookup",
1000 };
1001
1002 VLIB_NODE_FN (lookup_ip6_dst_itf_node) (vlib_main_t * vm,
1003                     vlib_node_runtime_t * node,
1004                     vlib_frame_t * from_frame)
1005 {
1006     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 1));
1007 }
1008
1009 VLIB_REGISTER_NODE (lookup_ip6_dst_itf_node) = {
1010     .name = "lookup-ip6-dst-itf",
1011     .vector_size = sizeof (u32),
1012     .format_trace = format_lookup_trace,
1013     .sibling_of = "ip6-lookup",
1014 };
1015
1016 VLIB_NODE_FN (lookup_ip6_src_node) (vlib_main_t * vm,
1017                 vlib_node_runtime_t * node,
1018                 vlib_frame_t * from_frame)
1019 {
1020     return (lookup_dpo_ip6_inline(vm, node, from_frame, 1, 0));
1021 }
1022
1023 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
1024     .name = "lookup-ip6-src",
1025     .vector_size = sizeof (u32),
1026     .format_trace = format_lookup_trace,
1027     .sibling_of = "ip6-lookup",
1028 };
1029
1030 always_inline uword
1031 lookup_dpo_mpls_inline (vlib_main_t * vm,
1032                        vlib_node_runtime_t * node,
1033                        vlib_frame_t * from_frame,
1034                        int table_from_interface)
1035 {
1036     u32 n_left_from, next_index, * from, * to_next;
1037     u32 thread_index = vlib_get_thread_index();
1038     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
1039
1040     from = vlib_frame_vector_args (from_frame);
1041     n_left_from = from_frame->n_vectors;
1042
1043     next_index = node->cached_next_index;
1044
1045     while (n_left_from > 0)
1046     {
1047         u32 n_left_to_next;
1048
1049         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
1050
1051         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1052         /*   } */
1053
1054         while (n_left_from > 0 && n_left_to_next > 0)
1055         {
1056             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash0;
1057             const mpls_unicast_header_t * hdr0;
1058             const load_balance_t *lb0;
1059             const lookup_dpo_t * lkd0;
1060             const dpo_id_t *dpo0;
1061             vlib_buffer_t * b0;
1062
1063             bi0 = from[0];
1064             to_next[0] = bi0;
1065             from += 1;
1066             to_next += 1;
1067             n_left_from -= 1;
1068             n_left_to_next -= 1;
1069
1070             b0 = vlib_get_buffer (vm, bi0);
1071             hdr0 = vlib_buffer_get_current (b0);
1072
1073             /* dst lookup was done by mpls lookup */
1074             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1075             lkd0 = lookup_dpo_get(lkdi0);
1076
1077             /*
1078              * choose between a lookup using the fib index in the DPO
1079              * or getting the FIB index from the interface.
1080              */
1081             if (table_from_interface)
1082             {
1083                 fib_index0 =
1084                     mpls_fib_table_get_index_for_sw_if_index(
1085                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1086             }
1087             else
1088             {
1089                 fib_index0 = lkd0->lkd_fib_index;
1090             }
1091
1092             /* do lookup */
1093             lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
1094             lb0  = load_balance_get(lbi0);
1095             dpo0 = load_balance_get_bucket_i(lb0, 0);
1096
1097             next0 = dpo0->dpoi_next_node;
1098             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1099
1100
1101             if (MPLS_IS_REPLICATE & lbi0)
1102             {
1103                 next0 = mpls_lookup_to_replicate_edge;
1104                 vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
1105                     (lbi0 & ~MPLS_IS_REPLICATE);
1106             }
1107             else
1108             {
1109                 lb0 = load_balance_get(lbi0);
1110                 ASSERT (lb0->lb_n_buckets > 0);
1111                 ASSERT (is_pow2 (lb0->lb_n_buckets));
1112
1113                 if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
1114                 {
1115                     hash0 = vnet_buffer (b0)->ip.flow_hash =
1116                         mpls_compute_flow_hash(hdr0, lb0->lb_hash_config);
1117                     dpo0 = load_balance_get_fwd_bucket
1118                         (lb0,
1119                          (hash0 & (lb0->lb_n_buckets_minus_1)));
1120                 }
1121                 else
1122                 {
1123                     dpo0 = load_balance_get_bucket_i (lb0, 0);
1124                 }
1125                 next0 = dpo0->dpoi_next_node;
1126
1127                 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1128
1129                 vlib_increment_combined_counter
1130                     (cm, thread_index, lbi0, 1,
1131                      vlib_buffer_length_in_chain (vm, b0));
1132             }
1133
1134             vnet_buffer (b0)->mpls.ttl = ((char*)hdr0)[3];
1135             vnet_buffer (b0)->mpls.exp = (((char*)hdr0)[2] & 0xe) >> 1;
1136             vnet_buffer (b0)->mpls.first = 1;
1137             vlib_buffer_advance(b0, sizeof(*hdr0));
1138
1139             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
1140                 vnet_buffer2(b0)->loop_counter = 0;
1141                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
1142             }
1143
1144             vnet_buffer2(b0)->loop_counter++;
1145
1146             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
1147                 next0 = MPLS_LOOKUP_NEXT_DROP;
1148
1149             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1150             {
1151                 lookup_trace_t *tr = vlib_add_trace (vm, node,
1152                                                      b0, sizeof (*tr));
1153                 tr->fib_index = fib_index0;
1154                 tr->lbi = lbi0;
1155                 tr->hdr = *hdr0;
1156             }
1157
1158            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1159                                             n_left_to_next, bi0, next0);
1160         }
1161         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1162     }
1163     return from_frame->n_vectors;
1164 }
1165
1166 static u8 *
1167 format_lookup_mpls_trace (u8 * s, va_list * args)
1168 {
1169     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1170     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1171     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
1172     u32 indent = format_get_indent (s);
1173     mpls_unicast_header_t hdr;
1174
1175     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
1176
1177     s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
1178                 format_white_space, indent,
1179                 t->fib_index,
1180                 format_mpls_header, hdr,
1181                 t->lbi);
1182     return s;
1183 }
1184
1185 VLIB_NODE_FN (lookup_mpls_dst_node) (vlib_main_t * vm,
1186                 vlib_node_runtime_t * node,
1187                 vlib_frame_t * from_frame)
1188 {
1189     return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
1190 }
1191
1192 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
1193     .name = "lookup-mpls-dst",
1194     .vector_size = sizeof (u32),
1195     .sibling_of = "mpls-lookup",
1196     .format_trace = format_lookup_mpls_trace,
1197     .n_next_nodes = 0,
1198 };
1199
1200 VLIB_NODE_FN (lookup_mpls_dst_itf_node) (vlib_main_t * vm,
1201                     vlib_node_runtime_t * node,
1202                     vlib_frame_t * from_frame)
1203 {
1204     return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
1205 }
1206
1207 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
1208     .name = "lookup-mpls-dst-itf",
1209     .vector_size = sizeof (u32),
1210     .sibling_of = "mpls-lookup",
1211     .format_trace = format_lookup_mpls_trace,
1212     .n_next_nodes = 0,
1213 };
1214
1215 typedef enum lookup_ip_dst_mcast_next_t_ {
1216     LOOKUP_IP_DST_MCAST_NEXT_DROP,
1217     LOOKUP_IP_DST_MCAST_NEXT_RPF,
1218     LOOKUP_IP_DST_MCAST_N_NEXT,
1219 } mfib_forward_lookup_next_t;
1220
1221 always_inline uword
1222 lookup_dpo_ip_dst_mcast_inline (vlib_main_t * vm,
1223                                 vlib_node_runtime_t * node,
1224                                 vlib_frame_t * from_frame,
1225                                 int is_v4)
1226 {
1227     u32 n_left_from, next_index, * from, * to_next;
1228
1229     from = vlib_frame_vector_args (from_frame);
1230     n_left_from = from_frame->n_vectors;
1231
1232     next_index = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1233
1234     while (n_left_from > 0)
1235     {
1236         u32 n_left_to_next;
1237
1238         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
1239
1240         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1241         /*   } */
1242
1243         while (n_left_from > 0 && n_left_to_next > 0)
1244         {
1245             u32 bi0, lkdi0, fib_index0,  next0;
1246             const lookup_dpo_t * lkd0;
1247             fib_node_index_t mfei0;
1248             vlib_buffer_t * b0;
1249
1250             bi0 = from[0];
1251             to_next[0] = bi0;
1252             from += 1;
1253             to_next += 1;
1254             n_left_from -= 1;
1255             n_left_to_next -= 1;
1256
1257             b0 = vlib_get_buffer (vm, bi0);
1258
1259             /* dst lookup was done by mpls lookup */
1260             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1261             lkd0 = lookup_dpo_get(lkdi0);
1262             fib_index0 = lkd0->lkd_fib_index;
1263             next0 = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1264
1265             if (is_v4)
1266             {
1267                 ip4_header_t * ip0;
1268
1269                 ip0 = vlib_buffer_get_current (b0);
1270                 mfei0 = ip4_mfib_table_lookup(ip4_mfib_get(fib_index0),
1271                                               &ip0->src_address,
1272                                               &ip0->dst_address,
1273                                               64);
1274                 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1275                 {
1276                     lookup_trace_t *tr = vlib_add_trace (vm, node,
1277                                                          b0, sizeof (*tr));
1278                     tr->fib_index = fib_index0;
1279                     tr->lbi = mfei0;
1280                     tr->addr.ip4 = ip0->dst_address;
1281                 }
1282             }
1283             else
1284             {
1285                 ip6_header_t * ip0;
1286
1287                 ip0 = vlib_buffer_get_current (b0);
1288                 mfei0 = ip6_mfib_table_fwd_lookup(ip6_mfib_get(fib_index0),
1289                                                   &ip0->src_address,
1290                                                   &ip0->dst_address);
1291                 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1292                 {
1293                     lookup_trace_t *tr = vlib_add_trace (vm, node,
1294                                                          b0, sizeof (*tr));
1295                     tr->fib_index = fib_index0;
1296                     tr->lbi = mfei0;
1297                     tr->addr.ip6 = ip0->dst_address;
1298                 }
1299             }
1300
1301             vnet_buffer (b0)->ip.adj_index[VLIB_TX] = mfei0;
1302
1303             if (!(b0->flags & VNET_BUFFER_F_LOOP_COUNTER_VALID)) {
1304                 vnet_buffer2(b0)->loop_counter = 0;
1305                 b0->flags |= VNET_BUFFER_F_LOOP_COUNTER_VALID;
1306             }
1307
1308             vnet_buffer2(b0)->loop_counter++;
1309
1310             if (PREDICT_FALSE(vnet_buffer2(b0)->loop_counter > MAX_LUKPS_PER_PACKET))
1311                 next0 = LOOKUP_IP_DST_MCAST_NEXT_DROP;
1312
1313             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1314                                             n_left_to_next, bi0, next0);
1315         }
1316         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1317     }
1318     return from_frame->n_vectors;
1319 }
1320
1321 VLIB_NODE_FN (lookup_ip4_dst_mcast_node) (vlib_main_t * vm,
1322                       vlib_node_runtime_t * node,
1323                       vlib_frame_t * from_frame)
1324 {
1325     return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 1));
1326 }
1327
1328 VLIB_REGISTER_NODE (lookup_ip4_dst_mcast_node) = {
1329     .name = "lookup-ip4-dst-mcast",
1330     .vector_size = sizeof (u32),
1331
1332     .format_trace = format_lookup_trace,
1333     .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1334     .next_nodes = {
1335         [LOOKUP_IP_DST_MCAST_NEXT_DROP] = "ip4-drop",
1336         [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip4-mfib-forward-rpf",
1337     },
1338 };
1339
1340 VLIB_NODE_FN (lookup_ip6_dst_mcast_node) (vlib_main_t * vm,
1341                       vlib_node_runtime_t * node,
1342                       vlib_frame_t * from_frame)
1343 {
1344     return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 0));
1345 }
1346
1347 VLIB_REGISTER_NODE (lookup_ip6_dst_mcast_node) = {
1348     .name = "lookup-ip6-dst-mcast",
1349     .vector_size = sizeof (u32),
1350
1351     .format_trace = format_lookup_trace,
1352     .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1353     .next_nodes = {
1354         [LOOKUP_IP_DST_MCAST_NEXT_DROP] = "ip6-drop",
1355         [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip6-mfib-forward-rpf",
1356     },
1357 };
1358
1359 static void
1360 lookup_dpo_mem_show (void)
1361 {
1362     fib_show_memory_usage("Lookup",
1363                           pool_elts(lookup_dpo_pool),
1364                           pool_len(lookup_dpo_pool),
1365                           sizeof(lookup_dpo_t));
1366 }
1367
1368 const static dpo_vft_t lkd_vft = {
1369     .dv_lock = lookup_dpo_lock,
1370     .dv_unlock = lookup_dpo_unlock,
1371     .dv_format = format_lookup_dpo,
1372 };
1373 const static dpo_vft_t lkd_vft_w_mem_show = {
1374     .dv_lock = lookup_dpo_lock,
1375     .dv_unlock = lookup_dpo_unlock,
1376     .dv_format = format_lookup_dpo,
1377     .dv_mem_show = lookup_dpo_mem_show,
1378 };
1379
1380 const static char* const lookup_src_ip4_nodes[] =
1381 {
1382     "lookup-ip4-src",
1383     NULL,
1384 };
1385 const static char* const lookup_src_ip6_nodes[] =
1386 {
1387     "lookup-ip6-src",
1388     NULL,
1389 };
1390 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
1391 {
1392     [DPO_PROTO_IP4]  = lookup_src_ip4_nodes,
1393     [DPO_PROTO_IP6]  = lookup_src_ip6_nodes,
1394     [DPO_PROTO_MPLS] = NULL,
1395 };
1396
1397 const static char* const lookup_dst_ip4_nodes[] =
1398 {
1399     "lookup-ip4-dst",
1400     NULL,
1401 };
1402 const static char* const lookup_dst_ip6_nodes[] =
1403 {
1404     "lookup-ip6-dst",
1405     NULL,
1406 };
1407 const static char* const lookup_dst_mpls_nodes[] =
1408 {
1409     "lookup-mpls-dst",
1410     NULL,
1411 };
1412 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
1413 {
1414     [DPO_PROTO_IP4]  = lookup_dst_ip4_nodes,
1415     [DPO_PROTO_IP6]  = lookup_dst_ip6_nodes,
1416     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
1417 };
1418
1419 const static char* const lookup_dst_mcast_ip4_nodes[] =
1420 {
1421     "lookup-ip4-dst-mcast",
1422     NULL,
1423 };
1424 const static char* const lookup_dst_mcast_ip6_nodes[] =
1425 {
1426     "lookup-ip6-dst-mcast",
1427     NULL,
1428 };
1429 const static char* const * const lookup_dst_mcast_nodes[DPO_PROTO_NUM] =
1430 {
1431     [DPO_PROTO_IP4]  = lookup_dst_mcast_ip4_nodes,
1432     [DPO_PROTO_IP6]  = lookup_dst_mcast_ip6_nodes,
1433 };
1434
1435 const static char* const lookup_dst_from_interface_ip4_nodes[] =
1436 {
1437     "lookup-ip4-dst-itf",
1438     NULL,
1439 };
1440 const static char* const lookup_dst_from_interface_ip6_nodes[] =
1441 {
1442     "lookup-ip6-dst-itf",
1443     NULL,
1444 };
1445 const static char* const lookup_dst_from_interface_mpls_nodes[] =
1446 {
1447     "lookup-mpls-dst-itf",
1448     NULL,
1449 };
1450 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
1451 {
1452     [DPO_PROTO_IP4]  = lookup_dst_from_interface_ip4_nodes,
1453     [DPO_PROTO_IP6]  = lookup_dst_from_interface_ip6_nodes,
1454     [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
1455 };
1456
1457 static clib_error_t *
1458 lookup_dpo_show (vlib_main_t * vm,
1459                  unformat_input_t * input,
1460                  vlib_cli_command_t * cmd)
1461 {
1462     index_t lkdi = INDEX_INVALID;
1463
1464     while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1465     {
1466         if (unformat (input, "%d", &lkdi))
1467             ;
1468         else
1469             break;
1470     }
1471
1472     if (INDEX_INVALID != lkdi)
1473     {
1474         if (pool_is_free_index(lookup_dpo_pool, lkdi))
1475                 vlib_cli_output (vm, "no such index %d", lkdi);
1476         else
1477                 vlib_cli_output (vm, "%U", format_lookup_dpo, lkdi);
1478     }
1479     else
1480     {
1481         lookup_dpo_t *lkd;
1482
1483         pool_foreach (lkd, lookup_dpo_pool)
1484          {
1485             vlib_cli_output (vm, "[@%d] %U",
1486                              lookup_dpo_get_index(lkd),
1487                              format_lookup_dpo,
1488                              lookup_dpo_get_index(lkd));
1489         }
1490     }
1491
1492     return 0;
1493 }
1494
1495 VLIB_CLI_COMMAND (replicate_show_command, static) = {
1496     .path = "show lookup-dpo",
1497     .short_help = "show lookup-dpo [<index>]",
1498     .function = lookup_dpo_show,
1499 };
1500
1501 void
1502 lookup_dpo_module_init (void)
1503 {
1504     dpo_register(DPO_LOOKUP, &lkd_vft_w_mem_show, NULL);
1505
1506     /*
1507      * There are various sorts of lookup; src or dst addr v4 /v6 etc.
1508      * there isn't an object type for each (there is only the lookup_dpo_t),
1509      * but, for performance reasons, there is a data plane function, and hence
1510      * VLIB node for each. VLIB graph node construction is based on DPO types
1511      * so we create sub-types.
1512      */
1513     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
1514         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
1515     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
1516         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
1517     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST] =
1518         dpo_register_new_type(&lkd_vft, lookup_dst_mcast_nodes);
1519     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
1520         dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes);
1521 }