vnet/vnet/dpo/lookup_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/lookup_dpo.h>
  18 #include <vnet/dpo/load_balance.h>
  19 #include <vnet/mpls/mpls.h>
  20 #include <vnet/fib/fib_table.h>
  21 #include <vnet/fib/ip4_fib.h>
  22 #include <vnet/fib/ip6_fib.h>
  23 #include <vnet/fib/mpls_fib.h>
  24
  25 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
  26
  27 /**
  28  * @brief Enumeration of the lookup subtypes
  29  */
  30 typedef enum lookup_sub_type_t_
  31 {
  32     LOOKUP_SUB_TYPE_SRC,
  33     LOOKUP_SUB_TYPE_DST,
  34     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
  35 } lookup_sub_type_t;
  36 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
  37
  38 #define FOR_EACH_LOOKUP_SUB_TYPE(_st)                                   \
  39     for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
  40
  41 /**
  42  * @brief pool of all MPLS Label DPOs
  43  */
  44 lookup_dpo_t *lookup_dpo_pool;
  45
  46 /**
  47  * @brief An array of registered DPO type values for the sub-types
  48  */
  49 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
  50
  51 static lookup_dpo_t *
  52 lookup_dpo_alloc (void)
  53 {
  54     lookup_dpo_t *lkd;
  55
  56     pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
  57
  58     return (lkd);
  59 }
  60
  61 static index_t
  62 lookup_dpo_get_index (lookup_dpo_t *lkd)
  63 {
  64     return (lkd - lookup_dpo_pool);
  65 }
  66
  67 static void
  68 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
  69                           dpo_proto_t proto,
  70                           lookup_input_t input,
  71                           lookup_table_t table_config,
  72                           dpo_id_t *dpo)
  73 {
  74     lookup_dpo_t *lkd;
  75     dpo_type_t type;
  76
  77     lkd = lookup_dpo_alloc();
  78     lkd->lkd_fib_index = fib_index;
  79     lkd->lkd_proto = proto;
  80     lkd->lkd_input = input;
  81     lkd->lkd_table = table_config;
  82
  83     /*
  84      * use the input type to select the lookup sub-type
  85      */
  86     type = 0;
  87
  88     switch (input)
  89     {
  90     case LOOKUP_INPUT_SRC_ADDR:
  91         type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
  92         break;
  93     case LOOKUP_INPUT_DST_ADDR:
  94         switch (table_config)
  95         {
  96         case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
  97             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
  98             break;
  99         case LOOKUP_TABLE_FROM_CONFIG:
 100             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
 101             break;
 102         }
 103     }
 104
 105     if (0 == type)
 106     {
 107         dpo_reset(dpo);
 108     }
 109     else
 110     {
 111         dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
 112     }
 113 }
 114
 115 void
 116 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
 117                                     dpo_proto_t proto,
 118                                     lookup_input_t input,
 119                                     lookup_table_t table_config,
 120                                     dpo_id_t *dpo)
 121 {
 122     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 123     {
 124         fib_table_lock(fib_index, dpo_proto_to_fib(proto));
 125     }
 126     lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
 127 }
 128
 129 void
 130 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 131                                    dpo_proto_t proto,
 132                                    lookup_input_t input,
 133                                    lookup_table_t table_config,
 134                                    dpo_id_t *dpo)
 135 {
 136     fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
 137
 138     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 139     {
 140         fib_index =
 141             fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 142                                               table_id);
 143     }
 144
 145     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
 146     lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
 147 }
 148
 149 u8*
 150 format_lookup_dpo (u8 *s, va_list *args)
 151 {
 152     index_t index = va_arg (*args, index_t);
 153     lookup_dpo_t *lkd;
 154
 155     lkd = lookup_dpo_get(index);
 156
 157     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
 158     {
 159         s = format(s, "%s lookup in interface's %U table",
 160                    lookup_input_names[lkd->lkd_input],
 161                    format_dpo_proto, lkd->lkd_proto);
 162     }
 163     else
 164     {
 165         s = format(s, "%s lookup in %U",
 166                    lookup_input_names[lkd->lkd_input],
 167                    format_fib_table_name, lkd->lkd_fib_index,
 168                    dpo_proto_to_fib(lkd->lkd_proto));
 169     }
 170     return (s);
 171 }
 172
 173 static void
 174 lookup_dpo_lock (dpo_id_t *dpo)
 175 {
 176     lookup_dpo_t *lkd;
 177
 178     lkd = lookup_dpo_get(dpo->dpoi_index);
 179
 180     lkd->lkd_locks++;
 181 }
 182
 183 static void
 184 lookup_dpo_unlock (dpo_id_t *dpo)
 185 {
 186     lookup_dpo_t *lkd;
 187
 188     lkd = lookup_dpo_get(dpo->dpoi_index);
 189
 190     lkd->lkd_locks--;
 191
 192     if (0 == lkd->lkd_locks)
 193     {
 194         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
 195         {
 196             fib_table_unlock(lkd->lkd_fib_index,
 197                              dpo_proto_to_fib(lkd->lkd_proto));
 198         }
 199         pool_put(lookup_dpo_pool, lkd);
 200     }
 201 }
 202
 203 always_inline void
 204 ip4_src_fib_lookup_one (u32 src_fib_index0,
 205                         const ip4_address_t * addr0,
 206                         u32 * src_adj_index0)
 207 {
 208     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 209     ip4_fib_mtrie_t * mtrie0;
 210
 211     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 212
 213     leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 214     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 0);
 215     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
 216     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 217     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 218
 219     /* Handle default route. */
 220     leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 221     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 222 }
 223
 224 always_inline void
 225 ip4_src_fib_lookup_two (u32 src_fib_index0,
 226                         u32 src_fib_index1,
 227                         const ip4_address_t * addr0,
 228                         const ip4_address_t * addr1,
 229                         u32 * src_adj_index0,
 230                         u32 * src_adj_index1)
 231 {
 232     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 233     ip4_fib_mtrie_t * mtrie0, * mtrie1;
 234
 235     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 236     mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
 237
 238     leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 239
 240     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 0);
 241     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 0);
 242
 243     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
 244     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 1);
 245
 246     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 247     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
 248
 249     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 250     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
 251
 252     /* Handle default route. */
 253     leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 254     leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
 255     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 256     src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 257 }
 258
 259 /**
 260  * @brief Lookup trace  data
 261  */
 262 typedef struct lookup_trace_t_
 263 {
 264     union {
 265         ip46_address_t addr;
 266         mpls_unicast_header_t hdr;
 267     };
 268     fib_node_index_t fib_index;
 269     index_t lbi;
 270 } lookup_trace_t;
 271
 272
 273 always_inline uword
 274 lookup_dpo_ip4_inline (vlib_main_t * vm,
 275                        vlib_node_runtime_t * node,
 276                        vlib_frame_t * from_frame,
 277                        int input_src_addr,
 278                        int table_from_interface)
 279 {
 280     u32 n_left_from, next_index, * from, * to_next;
 281     u32 cpu_index = os_get_cpu_number();
 282     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 283
 284     from = vlib_frame_vector_args (from_frame);
 285     n_left_from = from_frame->n_vectors;
 286
 287     next_index = node->cached_next_index;
 288
 289     while (n_left_from > 0)
 290     {
 291         u32 n_left_to_next;
 292
 293         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 294
 295         while (n_left_from > 0 && n_left_to_next > 0)
 296         {
 297             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 298             flow_hash_config_t flow_hash_config0;
 299             const ip4_address_t *input_addr;
 300             const load_balance_t *lb0;
 301             const lookup_dpo_t * lkd0;
 302             const ip4_header_t * ip0;
 303             const dpo_id_t *dpo0;
 304             vlib_buffer_t * b0;
 305
 306             bi0 = from[0];
 307             to_next[0] = bi0;
 308             from += 1;
 309             to_next += 1;
 310             n_left_from -= 1;
 311             n_left_to_next -= 1;
 312
 313             b0 = vlib_get_buffer (vm, bi0);
 314             ip0 = vlib_buffer_get_current (b0);
 315
 316             /* dst lookup was done by ip4 lookup */
 317             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 318             lkd0 = lookup_dpo_get(lkdi0);
 319
 320             /*
 321              * choose between a lookup using the fib index in the DPO
 322              * or getting the FIB index from the interface.
 323              */
 324             if (table_from_interface)
 325             {
 326                 fib_index0 =
 327                     ip4_fib_table_get_index_for_sw_if_index(
 328                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 329             }
 330             else
 331             {
 332                 fib_index0 = lkd0->lkd_fib_index;
 333             }
 334
 335             /*
 336              * choose between a source or destination address lookup in the table
 337              */
 338             if (input_src_addr)
 339             {
 340                 input_addr = &ip0->src_address;
 341             }
 342             else
 343             {
 344                 input_addr = &ip0->dst_address;
 345             }
 346
 347             /* do lookup */
 348             ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
 349             lb0 = load_balance_get(lbi0);
 350
 351             /* Use flow hash to compute multipath adjacency. */
 352             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 353
 354             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 355             {
 356                 flow_hash_config0 = lb0->lb_hash_config;
 357                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 358                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 359             }
 360
 361             dpo0 = load_balance_get_bucket_i(lb0,
 362                                              (hash_c0 &
 363                                               (lb0->lb_n_buckets_minus_1)));
 364
 365             next0 = dpo0->dpoi_next_node;
 366             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 367
 368             vlib_increment_combined_counter
 369                 (cm, cpu_index, lbi0, 1,
 370                  vlib_buffer_length_in_chain (vm, b0));
 371
 372             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 373             {
 374                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 375                                                      b0, sizeof (*tr));
 376                 tr->fib_index = fib_index0;
 377                 tr->lbi = lbi0;
 378                 tr->addr.ip4 = *input_addr;
 379             }
 380
 381             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 382                                             n_left_to_next, bi0, next0);
 383         }
 384         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 385     }
 386     return from_frame->n_vectors;
 387 }
 388
 389 static u8 *
 390 format_lookup_trace (u8 * s, va_list * args)
 391 {
 392     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 393     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 394     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 395     uword indent = format_get_indent (s);
 396     s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
 397                 format_white_space, indent,
 398                 t->fib_index,
 399                 format_ip46_address, &t->addr, IP46_TYPE_ANY,
 400                 t->lbi);
 401     return s;
 402 }
 403
 404 always_inline uword
 405 lookup_ip4_dst (vlib_main_t * vm,
 406                 vlib_node_runtime_t * node,
 407                 vlib_frame_t * from_frame)
 408 {
 409     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
 410 }
 411
 412 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
 413     .function = lookup_ip4_dst,
 414     .name = "lookup-ip4-dst",
 415     .vector_size = sizeof (u32),
 416     .sibling_of = "ip4-lookup",
 417     .format_trace = format_lookup_trace,
 418 };
 419 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_node, lookup_ip4_dst)
 420
 421 always_inline uword
 422 lookup_ip4_dst_itf (vlib_main_t * vm,
 423                     vlib_node_runtime_t * node,
 424                     vlib_frame_t * from_frame)
 425 {
 426     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
 427 }
 428
 429 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
 430     .function = lookup_ip4_dst_itf,
 431     .name = "lookup-ip4-dst-itf",
 432     .vector_size = sizeof (u32),
 433     .sibling_of = "ip4-lookup",
 434     .format_trace = format_lookup_trace,
 435 };
 436 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_itf_node, lookup_ip4_dst_itf)
 437
 438 always_inline uword
 439 lookup_ip4_src (vlib_main_t * vm,
 440                 vlib_node_runtime_t * node,
 441                 vlib_frame_t * from_frame)
 442 {
 443     return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
 444 }
 445
 446 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
 447     .function = lookup_ip4_src,
 448     .name = "lookup-ip4-src",
 449     .vector_size = sizeof (u32),
 450     .format_trace = format_lookup_trace,
 451     .sibling_of = "ip4-lookup",
 452 };
 453 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_src_node, lookup_ip4_src)
 454
 455 always_inline uword
 456 lookup_dpo_ip6_inline (vlib_main_t * vm,
 457                        vlib_node_runtime_t * node,
 458                        vlib_frame_t * from_frame,
 459                        int input_src_addr)
 460 {
 461     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 462     u32 n_left_from, next_index, * from, * to_next;
 463     u32 cpu_index = os_get_cpu_number();
 464
 465     from = vlib_frame_vector_args (from_frame);
 466     n_left_from = from_frame->n_vectors;
 467
 468     next_index = node->cached_next_index;
 469
 470     while (n_left_from > 0)
 471     {
 472         u32 n_left_to_next;
 473
 474         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 475
 476         while (n_left_from > 0 && n_left_to_next > 0)
 477         {
 478             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 479             flow_hash_config_t flow_hash_config0;
 480             const ip6_address_t *input_addr0;
 481             const load_balance_t *lb0;
 482             const lookup_dpo_t * lkd0;
 483             const ip6_header_t * ip0;
 484             const dpo_id_t *dpo0;
 485             vlib_buffer_t * b0;
 486
 487             bi0 = from[0];
 488             to_next[0] = bi0;
 489             from += 1;
 490             to_next += 1;
 491             n_left_from -= 1;
 492             n_left_to_next -= 1;
 493
 494             b0 = vlib_get_buffer (vm, bi0);
 495             ip0 = vlib_buffer_get_current (b0);
 496
 497             /* dst lookup was done by ip6 lookup */
 498             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 499             lkd0 = lookup_dpo_get(lkdi0);
 500             fib_index0 = lkd0->lkd_fib_index;
 501
 502             /*
 503              * choose between a source or destination address lookup in the table
 504              */
 505             if (input_src_addr)
 506             {
 507                 input_addr0 = &ip0->src_address;
 508             }
 509             else
 510             {
 511                 input_addr0 = &ip0->dst_address;
 512             }
 513
 514             /* do src lookup */
 515             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 516                                                fib_index0,
 517                                                input_addr0);
 518             lb0 = load_balance_get(lbi0);
 519
 520             /* Use flow hash to compute multipath adjacency. */
 521             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 522
 523             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 524             {
 525                 flow_hash_config0 = lb0->lb_hash_config;
 526                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 527                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 528             }
 529
 530             dpo0 = load_balance_get_bucket_i(lb0,
 531                                              (hash_c0 &
 532                                               (lb0->lb_n_buckets_minus_1)));
 533
 534             next0 = dpo0->dpoi_next_node;
 535             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 536
 537             vlib_increment_combined_counter
 538                 (cm, cpu_index, lbi0, 1,
 539                  vlib_buffer_length_in_chain (vm, b0));
 540
 541             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 542             {
 543                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 544                                                      b0, sizeof (*tr));
 545                 tr->fib_index = fib_index0;
 546                 tr->lbi = lbi0;
 547                 tr->addr.ip6 = *input_addr0;
 548             }
 549             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 550                                             n_left_to_next, bi0, next0);
 551         }
 552         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 553     }
 554     return from_frame->n_vectors;
 555 }
 556
 557 always_inline uword
 558 lookup_ip6_dst (vlib_main_t * vm,
 559                 vlib_node_runtime_t * node,
 560                 vlib_frame_t * from_frame)
 561 {
 562     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/));
 563 }
 564
 565 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
 566     .function = lookup_ip6_dst,
 567     .name = "lookup-ip6-dst",
 568     .vector_size = sizeof (u32),
 569     .format_trace = format_lookup_trace,
 570     .sibling_of = "ip6-lookup",
 571 };
 572 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_node, lookup_ip6_dst)
 573
 574 always_inline uword
 575 lookup_ip6_src (vlib_main_t * vm,
 576                 vlib_node_runtime_t * node,
 577                 vlib_frame_t * from_frame)
 578 {
 579     return (lookup_dpo_ip6_inline(vm, node, from_frame, 1 /*use src*/));
 580 }
 581
 582 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
 583     .function = lookup_ip6_src,
 584     .name = "lookup-ip6-src",
 585     .vector_size = sizeof (u32),
 586     .format_trace = format_lookup_trace,
 587     .sibling_of = "ip6-lookup",
 588 };
 589 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_src_node, lookup_ip6_src)
 590
 591 always_inline uword
 592 lookup_dpo_mpls_inline (vlib_main_t * vm,
 593                        vlib_node_runtime_t * node,
 594                        vlib_frame_t * from_frame,
 595                        int table_from_interface)
 596 {
 597     u32 n_left_from, next_index, * from, * to_next;
 598     u32 cpu_index = os_get_cpu_number();
 599     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 600
 601     from = vlib_frame_vector_args (from_frame);
 602     n_left_from = from_frame->n_vectors;
 603
 604     next_index = node->cached_next_index;
 605
 606     while (n_left_from > 0)
 607     {
 608         u32 n_left_to_next;
 609
 610         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 611
 612         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
 613         /*   } */
 614
 615         while (n_left_from > 0 && n_left_to_next > 0)
 616         {
 617             u32 bi0, lkdi0, lbi0, fib_index0,  next0;
 618             const mpls_unicast_header_t * hdr0;
 619             const load_balance_t *lb0;
 620             const lookup_dpo_t * lkd0;
 621             const dpo_id_t *dpo0;
 622             vlib_buffer_t * b0;
 623
 624             bi0 = from[0];
 625             to_next[0] = bi0;
 626             from += 1;
 627             to_next += 1;
 628             n_left_from -= 1;
 629             n_left_to_next -= 1;
 630
 631             b0 = vlib_get_buffer (vm, bi0);
 632             hdr0 = vlib_buffer_get_current (b0);
 633
 634             /* dst lookup was done by mpls lookup */
 635             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 636             lkd0 = lookup_dpo_get(lkdi0);
 637
 638             /*
 639              * choose between a lookup using the fib index in the DPO
 640              * or getting the FIB index from the interface.
 641              */
 642             if (table_from_interface)
 643             {
 644                 fib_index0 =
 645                     mpls_fib_table_get_index_for_sw_if_index(
 646                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 647             }
 648             else
 649             {
 650                 fib_index0 = lkd0->lkd_fib_index;
 651             }
 652
 653             /* do lookup */
 654             lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
 655             lb0  = load_balance_get(lbi0);
 656             dpo0 = load_balance_get_bucket_i(lb0, 0);
 657
 658             next0 = dpo0->dpoi_next_node;
 659             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 660
 661             vlib_increment_combined_counter
 662                 (cm, cpu_index, lbi0, 1,
 663                  vlib_buffer_length_in_chain (vm, b0));
 664
 665             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 666             {
 667                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 668                                                      b0, sizeof (*tr));
 669                 tr->fib_index = fib_index0;
 670                 tr->lbi = lbi0;
 671                 tr->hdr = *hdr0;
 672             }
 673
 674            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 675                                             n_left_to_next, bi0, next0);
 676         }
 677         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 678     }
 679     return from_frame->n_vectors;
 680 }
 681
 682 static u8 *
 683 format_lookup_mpls_trace (u8 * s, va_list * args)
 684 {
 685     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 686     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 687     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 688     uword indent = format_get_indent (s);
 689     mpls_unicast_header_t hdr;
 690
 691     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
 692
 693     s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
 694                 format_white_space, indent,
 695                 t->fib_index,
 696                 format_mpls_header, hdr,
 697                 t->lbi);
 698     return s;
 699 }
 700
 701 always_inline uword
 702 lookup_mpls_dst (vlib_main_t * vm,
 703                 vlib_node_runtime_t * node,
 704                 vlib_frame_t * from_frame)
 705 {
 706     return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
 707 }
 708
 709 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
 710     .function = lookup_mpls_dst,
 711     .name = "lookup-mpls-dst",
 712     .vector_size = sizeof (u32),
 713     .sibling_of = "mpls-lookup",
 714     .format_trace = format_lookup_mpls_trace,
 715     .n_next_nodes = 0,
 716 };
 717 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_node, lookup_mpls_dst)
 718
 719 always_inline uword
 720 lookup_mpls_dst_itf (vlib_main_t * vm,
 721                     vlib_node_runtime_t * node,
 722                     vlib_frame_t * from_frame)
 723 {
 724     return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
 725 }
 726
 727 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
 728     .function = lookup_mpls_dst_itf,
 729     .name = "lookup-mpls-dst-itf",
 730     .vector_size = sizeof (u32),
 731     .sibling_of = "mpls-lookup",
 732     .format_trace = format_lookup_mpls_trace,
 733     .n_next_nodes = 0,
 734 };
 735 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf)
 736
 737 static void
 738 lookup_dpo_mem_show (void)
 739 {
 740     fib_show_memory_usage("Lookup",
 741                           pool_elts(lookup_dpo_pool),
 742                           pool_len(lookup_dpo_pool),
 743                           sizeof(lookup_dpo_t));
 744 }
 745
 746 const static dpo_vft_t lkd_vft = {
 747     .dv_lock = lookup_dpo_lock,
 748     .dv_unlock = lookup_dpo_unlock,
 749     .dv_format = format_lookup_dpo,
 750 };
 751 const static dpo_vft_t lkd_vft_w_mem_show = {
 752     .dv_lock = lookup_dpo_lock,
 753     .dv_unlock = lookup_dpo_unlock,
 754     .dv_format = format_lookup_dpo,
 755     .dv_mem_show = lookup_dpo_mem_show,
 756 };
 757
 758 const static char* const lookup_src_ip4_nodes[] =
 759 {
 760     "lookup-ip4-src",
 761     NULL,
 762 };
 763 const static char* const lookup_src_ip6_nodes[] =
 764 {
 765     "lookup-ip6-src",
 766     NULL,
 767 };
 768 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
 769 {
 770     [DPO_PROTO_IP4]  = lookup_src_ip4_nodes,
 771     [DPO_PROTO_IP6]  = lookup_src_ip6_nodes,
 772     [DPO_PROTO_MPLS] = NULL,
 773 };
 774
 775 const static char* const lookup_dst_ip4_nodes[] =
 776 {
 777     "lookup-ip4-dst",
 778     NULL,
 779 };
 780 const static char* const lookup_dst_ip6_nodes[] =
 781 {
 782     "lookup-ip6-dst",
 783     NULL,
 784 };
 785 const static char* const lookup_dst_mpls_nodes[] =
 786 {
 787     "lookup-mpls-dst",
 788     NULL,
 789 };
 790 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
 791 {
 792     [DPO_PROTO_IP4]  = lookup_dst_ip4_nodes,
 793     [DPO_PROTO_IP6]  = lookup_dst_ip6_nodes,
 794     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
 795 };
 796
 797 const static char* const lookup_dst_from_interface_ip4_nodes[] =
 798 {
 799     "lookup-ip4-dst-itf",
 800     NULL,
 801 };
 802 const static char* const lookup_dst_from_interface_ip6_nodes[] =
 803 {
 804     "lookup-ip6-dst-itf",
 805     NULL,
 806 };
 807 const static char* const lookup_dst_from_interface_mpls_nodes[] =
 808 {
 809     "lookup-mpls-dst-itf",
 810     NULL,
 811 };
 812 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
 813 {
 814     [DPO_PROTO_IP4]  = lookup_dst_from_interface_ip4_nodes,
 815     [DPO_PROTO_IP6]  = lookup_dst_from_interface_ip6_nodes,
 816     [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
 817 };
 818
 819
 820 void
 821 lookup_dpo_module_init (void)
 822 {
 823     dpo_register(DPO_LOOKUP, &lkd_vft_w_mem_show, NULL);
 824
 825     /*
 826      * There are various sorts of lookup; src or dst addr v4 /v6 etc.
 827      * there isn't an object type for each (there is only the lookup_dpo_t),
 828      * but, for performance reasons, there is a data plane function, and hence
 829      * VLIB node for each. VLIB graph node construction is based on DPO types
 830      * so we create sub-types.
 831      */
 832     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
 833         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
 834     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
 835         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
 836     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
 837         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
 838 }