vnet/vnet/dpo/lookup_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/lookup_dpo.h>
  18 #include <vnet/dpo/load_balance.h>
  19 #include <vnet/mpls/mpls.h>
  20 #include <vnet/fib/fib_table.h>
  21 #include <vnet/fib/ip4_fib.h>
  22 #include <vnet/fib/ip6_fib.h>
  23 #include <vnet/fib/mpls_fib.h>
  24
  25 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
  26
  27 /**
  28  * @brief Enumeration of the lookup subtypes
  29  */
  30 typedef enum lookup_sub_type_t_
  31 {
  32     LOOKUP_SUB_TYPE_SRC,
  33     LOOKUP_SUB_TYPE_DST,
  34     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
  35 } lookup_sub_type_t;
  36 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
  37
  38 #define FOR_EACH_LOOKUP_SUB_TYPE(_st)                                   \
  39     for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
  40
  41 /**
  42  * @brief pool of all MPLS Label DPOs
  43  */
  44 lookup_dpo_t *lookup_dpo_pool;
  45
  46 /**
  47  * @brief An array of registered DPO type values for the sub-types
  48  */
  49 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
  50
  51 static lookup_dpo_t *
  52 lookup_dpo_alloc (void)
  53 {
  54     lookup_dpo_t *lkd;
  55
  56     pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
  57
  58     return (lkd);
  59 }
  60
  61 static index_t
  62 lookup_dpo_get_index (lookup_dpo_t *lkd)
  63 {
  64     return (lkd - lookup_dpo_pool);
  65 }
  66
  67 static void
  68 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
  69                           dpo_proto_t proto,
  70                           lookup_input_t input,
  71                           lookup_table_t table_config,
  72                           dpo_id_t *dpo)
  73 {
  74     lookup_dpo_t *lkd;
  75     dpo_type_t type;
  76
  77     lkd = lookup_dpo_alloc();
  78     lkd->lkd_fib_index = fib_index;
  79     lkd->lkd_proto = proto;
  80     lkd->lkd_input = input;
  81     lkd->lkd_table = table_config;
  82
  83     /*
  84      * use the input type to select the lookup sub-type
  85      */
  86     type = 0;
  87
  88     switch (input)
  89     {
  90     case LOOKUP_INPUT_SRC_ADDR:
  91         type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
  92         break;
  93     case LOOKUP_INPUT_DST_ADDR:
  94         switch (table_config)
  95         {
  96         case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
  97             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
  98             break;
  99         case LOOKUP_TABLE_FROM_CONFIG:
 100             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
 101             break;
 102         }
 103     }
 104
 105     if (0 == type)
 106     {
 107         dpo_reset(dpo);
 108     }
 109     else
 110     {
 111         dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
 112     }
 113 }
 114
 115 void
 116 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
 117                                     dpo_proto_t proto,
 118                                     lookup_input_t input,
 119                                     lookup_table_t table_config,
 120                                     dpo_id_t *dpo)
 121 {
 122     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 123     {
 124         fib_table_lock(fib_index, dpo_proto_to_fib(proto));
 125     }
 126     lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
 127 }
 128
 129 void
 130 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 131                                    dpo_proto_t proto,
 132                                    lookup_input_t input,
 133                                    lookup_table_t table_config,
 134                                    dpo_id_t *dpo)
 135 {
 136     fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
 137
 138     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 139     {
 140         fib_index =
 141             fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 142                                               table_id);
 143     }
 144
 145     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
 146     lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
 147 }
 148
 149 u8*
 150 format_lookup_dpo (u8 *s, va_list *args)
 151 {
 152     index_t index = va_arg (*args, index_t);
 153     lookup_dpo_t *lkd;
 154
 155     lkd = lookup_dpo_get(index);
 156
 157     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
 158     {
 159         s = format(s, "%s lookup in interface's %U table",
 160                    lookup_input_names[lkd->lkd_input],
 161                    format_dpo_proto, lkd->lkd_proto);
 162     }
 163     else
 164     {
 165         s = format(s, "%s lookup in %U",
 166                    lookup_input_names[lkd->lkd_input],
 167                    format_fib_table_name, lkd->lkd_fib_index,
 168                    dpo_proto_to_fib(lkd->lkd_proto));
 169     }
 170     return (s);
 171 }
 172
 173 static void
 174 lookup_dpo_lock (dpo_id_t *dpo)
 175 {
 176     lookup_dpo_t *lkd;
 177
 178     lkd = lookup_dpo_get(dpo->dpoi_index);
 179
 180     lkd->lkd_locks++;
 181 }
 182
 183 static void
 184 lookup_dpo_unlock (dpo_id_t *dpo)
 185 {
 186     lookup_dpo_t *lkd;
 187
 188     lkd = lookup_dpo_get(dpo->dpoi_index);
 189
 190     lkd->lkd_locks--;
 191
 192     if (0 == lkd->lkd_locks)
 193     {
 194         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
 195         {
 196             fib_table_unlock(lkd->lkd_fib_index,
 197                              dpo_proto_to_fib(lkd->lkd_proto));
 198         }
 199         pool_put(lookup_dpo_pool, lkd);
 200     }
 201 }
 202
 203 always_inline void
 204 ip4_src_fib_lookup_one (u32 src_fib_index0,
 205                         const ip4_address_t * addr0,
 206                         u32 * src_adj_index0)
 207 {
 208     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 209     ip4_fib_mtrie_t * mtrie0;
 210
 211     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 212
 213     leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 214     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 0);
 215     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
 216     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 217     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 218
 219     /* Handle default route. */
 220     leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 221     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 222 }
 223
 224 always_inline void
 225 ip4_src_fib_lookup_two (u32 src_fib_index0,
 226                         u32 src_fib_index1,
 227                         const ip4_address_t * addr0,
 228                         const ip4_address_t * addr1,
 229                         u32 * src_adj_index0,
 230                         u32 * src_adj_index1)
 231 {
 232     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 233     ip4_fib_mtrie_t * mtrie0, * mtrie1;
 234
 235     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 236     mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
 237
 238     leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 239
 240     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 0);
 241     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 0);
 242
 243     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
 244     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 1);
 245
 246     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 247     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
 248
 249     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 250     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
 251
 252     /* Handle default route. */
 253     leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 254     leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
 255     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 256     src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 257 }
 258
 259 /**
 260  * @brief Lookup trace  data
 261  */
 262 typedef struct lookup_trace_t_
 263 {
 264     union {
 265         ip46_address_t addr;
 266         mpls_unicast_header_t hdr;
 267     };
 268     fib_node_index_t fib_index;
 269     index_t lbi;
 270 } lookup_trace_t;
 271
 272
 273 always_inline uword
 274 lookup_dpo_ip4_inline (vlib_main_t * vm,
 275                        vlib_node_runtime_t * node,
 276                        vlib_frame_t * from_frame,
 277                        int input_src_addr,
 278                        int table_from_interface)
 279 {
 280     u32 n_left_from, next_index, * from, * to_next;
 281     u32 cpu_index = os_get_cpu_number();
 282     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 283
 284     from = vlib_frame_vector_args (from_frame);
 285     n_left_from = from_frame->n_vectors;
 286
 287     next_index = node->cached_next_index;
 288
 289     while (n_left_from > 0)
 290     {
 291         u32 n_left_to_next;
 292
 293         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 294
 295         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
 296         /*   } */
 297
 298         while (n_left_from > 0 && n_left_to_next > 0)
 299         {
 300             u32 bi0, lkdi0, lbi0, fib_index0,  next0;
 301             const ip4_address_t *input_addr;
 302             const load_balance_t *lb0;
 303             const lookup_dpo_t * lkd0;
 304             const ip4_header_t * ip0;
 305             const dpo_id_t *dpo0;
 306             vlib_buffer_t * b0;
 307
 308             bi0 = from[0];
 309             to_next[0] = bi0;
 310             from += 1;
 311             to_next += 1;
 312             n_left_from -= 1;
 313             n_left_to_next -= 1;
 314
 315             b0 = vlib_get_buffer (vm, bi0);
 316             ip0 = vlib_buffer_get_current (b0);
 317
 318             /* dst lookup was done by ip4 lookup */
 319             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 320             lkd0 = lookup_dpo_get(lkdi0);
 321
 322             /*
 323              * choose between a lookup using the fib index in the DPO
 324              * or getting the FIB index from the interface.
 325              */
 326             if (table_from_interface)
 327             {
 328                 fib_index0 =
 329                     ip4_fib_table_get_index_for_sw_if_index(
 330                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 331             }
 332             else
 333             {
 334                 fib_index0 = lkd0->lkd_fib_index;
 335             }
 336
 337             /*
 338              * choose between a source or destination address lookup in the table
 339              */
 340             if (input_src_addr)
 341             {
 342                 input_addr = &ip0->src_address;
 343             }
 344             else
 345             {
 346                 input_addr = &ip0->dst_address;
 347             }
 348
 349             /* do lookup */
 350             ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
 351             lb0 = load_balance_get(lbi0);
 352             dpo0 = load_balance_get_bucket_i(lb0, 0);
 353
 354             next0 = dpo0->dpoi_next_node;
 355             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 356
 357             vlib_increment_combined_counter
 358                 (cm, cpu_index, lbi0, 1,
 359                  vlib_buffer_length_in_chain (vm, b0));
 360
 361             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 362             {
 363                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 364                                                      b0, sizeof (*tr));
 365                 tr->fib_index = fib_index0;
 366                 tr->lbi = lbi0;
 367                 tr->addr.ip4 = *input_addr;
 368             }
 369
 370             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 371                                             n_left_to_next, bi0, next0);
 372         }
 373         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 374     }
 375     return from_frame->n_vectors;
 376 }
 377
 378 static u8 *
 379 format_lookup_trace (u8 * s, va_list * args)
 380 {
 381     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 382     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 383     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 384     uword indent = format_get_indent (s);
 385     s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
 386                 format_white_space, indent,
 387                 t->fib_index,
 388                 format_ip46_address, &t->addr, IP46_TYPE_ANY,
 389                 t->lbi);
 390     return s;
 391 }
 392
 393 always_inline uword
 394 lookup_ip4_dst (vlib_main_t * vm,
 395                 vlib_node_runtime_t * node,
 396                 vlib_frame_t * from_frame)
 397 {
 398     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
 399 }
 400
 401 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
 402     .function = lookup_ip4_dst,
 403     .name = "lookup-ip4-dst",
 404     .vector_size = sizeof (u32),
 405     .sibling_of = "ip4-lookup",
 406     .format_trace = format_lookup_trace,
 407 };
 408 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_node, lookup_ip4_dst)
 409
 410 always_inline uword
 411 lookup_ip4_dst_itf (vlib_main_t * vm,
 412                     vlib_node_runtime_t * node,
 413                     vlib_frame_t * from_frame)
 414 {
 415     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
 416 }
 417
 418 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
 419     .function = lookup_ip4_dst_itf,
 420     .name = "lookup-ip4-dst-itf",
 421     .vector_size = sizeof (u32),
 422     .sibling_of = "ip4-lookup",
 423     .format_trace = format_lookup_trace,
 424 };
 425 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_itf_node, lookup_ip4_dst_itf)
 426
 427 always_inline uword
 428 lookup_ip4_src (vlib_main_t * vm,
 429                 vlib_node_runtime_t * node,
 430                 vlib_frame_t * from_frame)
 431 {
 432     return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
 433 }
 434
 435 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
 436     .function = lookup_ip4_src,
 437     .name = "lookup-ip4-src",
 438     .vector_size = sizeof (u32),
 439     .format_trace = format_lookup_trace,
 440     .sibling_of = "ip4-lookup",
 441 };
 442 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_src_node, lookup_ip4_src)
 443
 444 always_inline uword
 445 lookup_dpo_ip6_inline (vlib_main_t * vm,
 446                        vlib_node_runtime_t * node,
 447                        vlib_frame_t * from_frame,
 448                        int input_src_addr)
 449 {
 450     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 451     u32 n_left_from, next_index, * from, * to_next;
 452     u32 cpu_index = os_get_cpu_number();
 453
 454     from = vlib_frame_vector_args (from_frame);
 455     n_left_from = from_frame->n_vectors;
 456
 457     next_index = node->cached_next_index;
 458
 459     while (n_left_from > 0)
 460     {
 461         u32 n_left_to_next;
 462
 463         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 464
 465         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
 466         /*   { */
 467         /*   } */
 468
 469         while (n_left_from > 0 && n_left_to_next > 0)
 470         {
 471             u32 bi0, lkdi0, lbi0, fib_index0, next0;
 472             const ip6_address_t *input_addr0;
 473             const load_balance_t *lb0;
 474             const lookup_dpo_t * lkd0;
 475             const ip6_header_t * ip0;
 476             const dpo_id_t *dpo0;
 477             vlib_buffer_t * b0;
 478
 479             bi0 = from[0];
 480             to_next[0] = bi0;
 481             from += 1;
 482             to_next += 1;
 483             n_left_from -= 1;
 484             n_left_to_next -= 1;
 485
 486             b0 = vlib_get_buffer (vm, bi0);
 487             ip0 = vlib_buffer_get_current (b0);
 488
 489             /* dst lookup was done by ip6 lookup */
 490             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 491             lkd0 = lookup_dpo_get(lkdi0);
 492             fib_index0 = lkd0->lkd_fib_index;
 493
 494             /*
 495              * choose between a source or destination address lookup in the table
 496              */
 497             if (input_src_addr)
 498             {
 499                 input_addr0 = &ip0->src_address;
 500             }
 501             else
 502             {
 503                 input_addr0 = &ip0->dst_address;
 504             }
 505
 506             /* do src lookup */
 507             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 508                                                fib_index0,
 509                                                input_addr0);
 510             lb0 = load_balance_get(lbi0);
 511             dpo0 = load_balance_get_bucket_i(lb0, 0);
 512
 513             next0 = dpo0->dpoi_next_node;
 514             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 515
 516             vlib_increment_combined_counter
 517                 (cm, cpu_index, lbi0, 1,
 518                  vlib_buffer_length_in_chain (vm, b0));
 519
 520             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 521             {
 522                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 523                                                      b0, sizeof (*tr));
 524                 tr->fib_index = fib_index0;
 525                 tr->lbi = lbi0;
 526                 tr->addr.ip6 = *input_addr0;
 527             }
 528             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 529                                             n_left_to_next, bi0, next0);
 530         }
 531         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 532     }
 533     return from_frame->n_vectors;
 534 }
 535
 536 always_inline uword
 537 lookup_ip6_dst (vlib_main_t * vm,
 538                 vlib_node_runtime_t * node,
 539                 vlib_frame_t * from_frame)
 540 {
 541     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/));
 542 }
 543
 544 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
 545     .function = lookup_ip6_dst,
 546     .name = "lookup-ip6-dst",
 547     .vector_size = sizeof (u32),
 548     .format_trace = format_lookup_trace,
 549     .sibling_of = "ip6-lookup",
 550 };
 551 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_node, lookup_ip6_dst)
 552
 553 always_inline uword
 554 lookup_ip6_src (vlib_main_t * vm,
 555                 vlib_node_runtime_t * node,
 556                 vlib_frame_t * from_frame)
 557 {
 558     return (lookup_dpo_ip6_inline(vm, node, from_frame, 1 /*use src*/));
 559 }
 560
 561 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
 562     .function = lookup_ip6_src,
 563     .name = "lookup-ip6-src",
 564     .vector_size = sizeof (u32),
 565     .format_trace = format_lookup_trace,
 566     .sibling_of = "ip6-lookup",
 567 };
 568 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_src_node, lookup_ip6_src)
 569
 570 always_inline uword
 571 lookup_dpo_mpls_inline (vlib_main_t * vm,
 572                        vlib_node_runtime_t * node,
 573                        vlib_frame_t * from_frame,
 574                        int table_from_interface)
 575 {
 576     u32 n_left_from, next_index, * from, * to_next;
 577     u32 cpu_index = os_get_cpu_number();
 578     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 579
 580     from = vlib_frame_vector_args (from_frame);
 581     n_left_from = from_frame->n_vectors;
 582
 583     next_index = node->cached_next_index;
 584
 585     while (n_left_from > 0)
 586     {
 587         u32 n_left_to_next;
 588
 589         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 590
 591         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
 592         /*   } */
 593
 594         while (n_left_from > 0 && n_left_to_next > 0)
 595         {
 596             u32 bi0, lkdi0, lbi0, fib_index0,  next0;
 597             const mpls_unicast_header_t * hdr0;
 598             const load_balance_t *lb0;
 599             const lookup_dpo_t * lkd0;
 600             const dpo_id_t *dpo0;
 601             vlib_buffer_t * b0;
 602
 603             bi0 = from[0];
 604             to_next[0] = bi0;
 605             from += 1;
 606             to_next += 1;
 607             n_left_from -= 1;
 608             n_left_to_next -= 1;
 609
 610             b0 = vlib_get_buffer (vm, bi0);
 611             hdr0 = vlib_buffer_get_current (b0);
 612
 613             /* dst lookup was done by mpls lookup */
 614             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 615             lkd0 = lookup_dpo_get(lkdi0);
 616
 617             /*
 618              * choose between a lookup using the fib index in the DPO
 619              * or getting the FIB index from the interface.
 620              */
 621             if (table_from_interface)
 622             {
 623                 fib_index0 =
 624                     mpls_fib_table_get_index_for_sw_if_index(
 625                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 626             }
 627             else
 628             {
 629                 fib_index0 = lkd0->lkd_fib_index;
 630             }
 631
 632             /* do lookup */
 633             lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
 634             lb0  = load_balance_get(lbi0);
 635             dpo0 = load_balance_get_bucket_i(lb0, 0);
 636
 637             next0 = dpo0->dpoi_next_node;
 638             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 639
 640             vlib_increment_combined_counter
 641                 (cm, cpu_index, lbi0, 1,
 642                  vlib_buffer_length_in_chain (vm, b0));
 643
 644             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 645             {
 646                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 647                                                      b0, sizeof (*tr));
 648                 tr->fib_index = fib_index0;
 649                 tr->lbi = lbi0;
 650                 tr->hdr = *hdr0;
 651             }
 652
 653            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 654                                             n_left_to_next, bi0, next0);
 655         }
 656         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 657     }
 658     return from_frame->n_vectors;
 659 }
 660
 661 static u8 *
 662 format_lookup_mpls_trace (u8 * s, va_list * args)
 663 {
 664     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 665     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 666     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 667     uword indent = format_get_indent (s);
 668     mpls_unicast_header_t hdr;
 669
 670     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
 671
 672     s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
 673                 format_white_space, indent,
 674                 t->fib_index,
 675                 format_mpls_header, hdr,
 676                 t->lbi);
 677     return s;
 678 }
 679
 680 always_inline uword
 681 lookup_mpls_dst (vlib_main_t * vm,
 682                 vlib_node_runtime_t * node,
 683                 vlib_frame_t * from_frame)
 684 {
 685     return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
 686 }
 687
 688 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
 689     .function = lookup_mpls_dst,
 690     .name = "lookup-mpls-dst",
 691     .vector_size = sizeof (u32),
 692     .sibling_of = "mpls-lookup",
 693     .format_trace = format_lookup_mpls_trace,
 694     .n_next_nodes = 0,
 695 };
 696 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_node, lookup_mpls_dst)
 697
 698 always_inline uword
 699 lookup_mpls_dst_itf (vlib_main_t * vm,
 700                     vlib_node_runtime_t * node,
 701                     vlib_frame_t * from_frame)
 702 {
 703     return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
 704 }
 705
 706 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
 707     .function = lookup_mpls_dst_itf,
 708     .name = "lookup-mpls-dst-itf",
 709     .vector_size = sizeof (u32),
 710     .sibling_of = "mpls-lookup",
 711     .format_trace = format_lookup_mpls_trace,
 712     .n_next_nodes = 0,
 713 };
 714 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf)
 715
 716 const static dpo_vft_t lkd_vft = {
 717     .dv_lock = lookup_dpo_lock,
 718     .dv_unlock = lookup_dpo_unlock,
 719     .dv_format = format_lookup_dpo,
 720 };
 721
 722 const static char* const lookup_src_ip4_nodes[] =
 723 {
 724     "lookup-ip4-src",
 725     NULL,
 726 };
 727 const static char* const lookup_src_ip6_nodes[] =
 728 {
 729     "lookup-ip6-src",
 730     NULL,
 731 };
 732 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
 733 {
 734     [DPO_PROTO_IP4]  = lookup_src_ip4_nodes,
 735     [DPO_PROTO_IP6]  = lookup_src_ip6_nodes,
 736     [DPO_PROTO_MPLS] = NULL,
 737 };
 738
 739 const static char* const lookup_dst_ip4_nodes[] =
 740 {
 741     "lookup-ip4-dst",
 742     NULL,
 743 };
 744 const static char* const lookup_dst_ip6_nodes[] =
 745 {
 746     "lookup-ip6-dst",
 747     NULL,
 748 };
 749 const static char* const lookup_dst_mpls_nodes[] =
 750 {
 751     "lookup-mpls-dst",
 752     NULL,
 753 };
 754 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
 755 {
 756     [DPO_PROTO_IP4]  = lookup_dst_ip4_nodes,
 757     [DPO_PROTO_IP6]  = lookup_dst_ip6_nodes,
 758     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
 759 };
 760
 761 const static char* const lookup_dst_from_interface_ip4_nodes[] =
 762 {
 763     "lookup-ip4-dst-itf",
 764     NULL,
 765 };
 766 const static char* const lookup_dst_from_interface_ip6_nodes[] =
 767 {
 768     "lookup-ip6-dst-itf",
 769     NULL,
 770 };
 771 const static char* const lookup_dst_from_interface_mpls_nodes[] =
 772 {
 773     "lookup-mpls-dst-itf",
 774     NULL,
 775 };
 776 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
 777 {
 778     [DPO_PROTO_IP4]  = lookup_dst_from_interface_ip4_nodes,
 779     [DPO_PROTO_IP6]  = lookup_dst_from_interface_ip6_nodes,
 780     [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
 781 };
 782
 783
 784 void
 785 lookup_dpo_module_init (void)
 786 {
 787     dpo_register(DPO_LOOKUP, &lkd_vft, NULL);
 788
 789     /*
 790      * There are various sorts of lookup; src or dst addr v4 /v6 etc.
 791      * there isn't an object type for each (there is only the lookup_dpo_t),
 792      * but, for performance reasons, there is a data plane function, and hence
 793      * VLIB node for each. VLIB graph node construction is based on DPO types
 794      * so we create sub-types.
 795      */
 796     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
 797         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
 798     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
 799         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
 800     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
 801         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
 802 }