vnet/vnet/dpo/lookup_dpo.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/dpo/lookup_dpo.h>
  18 #include <vnet/dpo/load_balance.h>
  19 #include <vnet/mpls/mpls.h>
  20 #include <vnet/fib/fib_table.h>
  21 #include <vnet/fib/ip4_fib.h>
  22 #include <vnet/fib/ip6_fib.h>
  23 #include <vnet/fib/mpls_fib.h>
  24
  25 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
  26
  27 /**
  28  * @brief Enumeration of the lookup subtypes
  29  */
  30 typedef enum lookup_sub_type_t_
  31 {
  32     LOOKUP_SUB_TYPE_SRC,
  33     LOOKUP_SUB_TYPE_DST,
  34     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
  35 } lookup_sub_type_t;
  36 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
  37
  38 #define FOR_EACH_LOOKUP_SUB_TYPE(_st)                                   \
  39     for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
  40
  41 /**
  42  * @brief pool of all MPLS Label DPOs
  43  */
  44 lookup_dpo_t *lookup_dpo_pool;
  45
  46 /**
  47  * @brief An array of registered DPO type values for the sub-types
  48  */
  49 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
  50
  51 static lookup_dpo_t *
  52 lookup_dpo_alloc (void)
  53 {
  54     lookup_dpo_t *lkd;
  55
  56     pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
  57
  58     return (lkd);
  59 }
  60
  61 static index_t
  62 lookup_dpo_get_index (lookup_dpo_t *lkd)
  63 {
  64     return (lkd - lookup_dpo_pool);
  65 }
  66
  67 static void
  68 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
  69                           dpo_proto_t proto,
  70                           lookup_input_t input,
  71                           lookup_table_t table_config,
  72                           dpo_id_t *dpo)
  73 {
  74     lookup_dpo_t *lkd;
  75     dpo_type_t type;
  76
  77     lkd = lookup_dpo_alloc();
  78     lkd->lkd_fib_index = fib_index;
  79     lkd->lkd_proto = proto;
  80     lkd->lkd_input = input;
  81     lkd->lkd_table = table_config;
  82
  83     /*
  84      * use the input type to select the lookup sub-type
  85      */
  86     type = 0;
  87
  88     switch (input)
  89     {
  90     case LOOKUP_INPUT_SRC_ADDR:
  91         type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
  92         break;
  93     case LOOKUP_INPUT_DST_ADDR:
  94         switch (table_config)
  95         {
  96         case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
  97             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
  98             break;
  99         case LOOKUP_TABLE_FROM_CONFIG:
 100             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
 101             break;
 102         }
 103     }
 104
 105     if (0 == type)
 106     {
 107         dpo_reset(dpo);
 108     }
 109     else
 110     {
 111         dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
 112     }
 113 }
 114
 115 void
 116 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
 117                                     dpo_proto_t proto,
 118                                     lookup_input_t input,
 119                                     lookup_table_t table_config,
 120                                     dpo_id_t *dpo)
 121 {
 122     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 123     {
 124         fib_table_lock(fib_index, dpo_proto_to_fib(proto));
 125     }
 126     lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
 127 }
 128
 129 void
 130 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 131                                    dpo_proto_t proto,
 132                                    lookup_input_t input,
 133                                    lookup_table_t table_config,
 134                                    dpo_id_t *dpo)
 135 {
 136     fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
 137
 138     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
 139     {
 140         fib_index =
 141             fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
 142                                               table_id);
 143     }
 144
 145     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
 146     lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
 147 }
 148
 149 u8*
 150 format_lookup_dpo (u8 *s, va_list *args)
 151 {
 152     index_t index = va_arg (*args, index_t);
 153     lookup_dpo_t *lkd;
 154
 155     lkd = lookup_dpo_get(index);
 156
 157     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
 158     {
 159         s = format(s, "%s lookup in interface's %U table",
 160                    lookup_input_names[lkd->lkd_input],
 161                    format_dpo_proto, lkd->lkd_proto);
 162     }
 163     else
 164     {
 165         s = format(s, "%s lookup in %U",
 166                    lookup_input_names[lkd->lkd_input],
 167                    format_fib_table_name, lkd->lkd_fib_index,
 168                    dpo_proto_to_fib(lkd->lkd_proto));
 169     }
 170     return (s);
 171 }
 172
 173 static void
 174 lookup_dpo_lock (dpo_id_t *dpo)
 175 {
 176     lookup_dpo_t *lkd;
 177
 178     lkd = lookup_dpo_get(dpo->dpoi_index);
 179
 180     lkd->lkd_locks++;
 181 }
 182
 183 static void
 184 lookup_dpo_unlock (dpo_id_t *dpo)
 185 {
 186     lookup_dpo_t *lkd;
 187
 188     lkd = lookup_dpo_get(dpo->dpoi_index);
 189
 190     lkd->lkd_locks--;
 191
 192     if (0 == lkd->lkd_locks)
 193     {
 194         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
 195         {
 196             fib_table_unlock(lkd->lkd_fib_index,
 197                              dpo_proto_to_fib(lkd->lkd_proto));
 198         }
 199         pool_put(lookup_dpo_pool, lkd);
 200     }
 201 }
 202
 203 always_inline void
 204 ip4_src_fib_lookup_one (u32 src_fib_index0,
 205                         const ip4_address_t * addr0,
 206                         u32 * src_adj_index0)
 207 {
 208     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 209     ip4_fib_mtrie_t * mtrie0;
 210
 211     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 212
 213     leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 214     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 0);
 215     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
 216     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 217     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 218
 219     /* Handle default route. */
 220     leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 221     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 222 }
 223
 224 always_inline void
 225 ip4_src_fib_lookup_two (u32 src_fib_index0,
 226                         u32 src_fib_index1,
 227                         const ip4_address_t * addr0,
 228                         const ip4_address_t * addr1,
 229                         u32 * src_adj_index0,
 230                         u32 * src_adj_index1)
 231 {
 232     ip4_fib_mtrie_leaf_t leaf0, leaf1;
 233     ip4_fib_mtrie_t * mtrie0, * mtrie1;
 234
 235     mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
 236     mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
 237
 238     leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 239
 240     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 0);
 241     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 0);
 242
 243     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
 244     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 1);
 245
 246     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
 247     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
 248
 249     leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
 250     leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
 251
 252     /* Handle default route. */
 253     leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 254     leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
 255     src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 256     src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 257 }
 258
 259 /**
 260  * @brief Lookup trace  data
 261  */
 262 typedef struct lookup_trace_t_
 263 {
 264     union {
 265         ip46_address_t addr;
 266         mpls_unicast_header_t hdr;
 267     };
 268     fib_node_index_t fib_index;
 269     index_t lbi;
 270 } lookup_trace_t;
 271
 272
 273 always_inline uword
 274 lookup_dpo_ip4_inline (vlib_main_t * vm,
 275                        vlib_node_runtime_t * node,
 276                        vlib_frame_t * from_frame,
 277                        int input_src_addr,
 278                        int table_from_interface)
 279 {
 280     u32 n_left_from, next_index, * from, * to_next;
 281     u32 cpu_index = os_get_cpu_number();
 282     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 283
 284     from = vlib_frame_vector_args (from_frame);
 285     n_left_from = from_frame->n_vectors;
 286
 287     next_index = node->cached_next_index;
 288
 289     while (n_left_from > 0)
 290     {
 291         u32 n_left_to_next;
 292
 293         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 294
 295         while (n_left_from >= 4 && n_left_to_next > 2)
 296         {
 297             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 298             flow_hash_config_t flow_hash_config0;
 299             const ip4_address_t *input_addr0;
 300             const load_balance_t *lb0;
 301             const lookup_dpo_t * lkd0;
 302             const ip4_header_t * ip0;
 303             const dpo_id_t *dpo0;
 304             vlib_buffer_t * b0;
 305             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 306             flow_hash_config_t flow_hash_config1;
 307             const ip4_address_t *input_addr1;
 308             const load_balance_t *lb1;
 309             const lookup_dpo_t * lkd1;
 310             const ip4_header_t * ip1;
 311             const dpo_id_t *dpo1;
 312             vlib_buffer_t * b1;
 313
 314             /* Prefetch next iteration. */
 315             {
 316                 vlib_buffer_t * p2, * p3;
 317
 318                 p2 = vlib_get_buffer (vm, from[2]);
 319                 p3 = vlib_get_buffer (vm, from[3]);
 320
 321                 vlib_prefetch_buffer_header (p2, LOAD);
 322                 vlib_prefetch_buffer_header (p3, LOAD);
 323
 324                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 325                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 326             }
 327
 328             bi0 = from[0];
 329             to_next[0] = bi0;
 330             bi1 = from[1];
 331             to_next[1] = bi1;
 332             from += 2;
 333             to_next += 2;
 334             n_left_from -= 2;
 335             n_left_to_next -= 2;
 336
 337             b0 = vlib_get_buffer (vm, bi0);
 338             ip0 = vlib_buffer_get_current (b0);
 339             b1 = vlib_get_buffer (vm, bi1);
 340             ip1 = vlib_buffer_get_current (b1);
 341
 342             /* dst lookup was done by ip4 lookup */
 343             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 344             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 345             lkd0 = lookup_dpo_get(lkdi0);
 346             lkd1 = lookup_dpo_get(lkdi1);
 347
 348             /*
 349              * choose between a lookup using the fib index in the DPO
 350              * or getting the FIB index from the interface.
 351              */
 352             if (table_from_interface)
 353             {
 354                 fib_index0 =
 355                     ip4_fib_table_get_index_for_sw_if_index(
 356                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 357                 fib_index1 =
 358                     ip4_fib_table_get_index_for_sw_if_index(
 359                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 360             }
 361             else
 362             {
 363                 fib_index0 = lkd0->lkd_fib_index;
 364                 fib_index1 = lkd1->lkd_fib_index;
 365             }
 366
 367             /*
 368              * choose between a source or destination address lookup in the table
 369              */
 370             if (input_src_addr)
 371             {
 372                 input_addr0 = &ip0->src_address;
 373                 input_addr1 = &ip1->src_address;
 374             }
 375             else
 376             {
 377                 input_addr0 = &ip0->dst_address;
 378                 input_addr1 = &ip1->dst_address;
 379             }
 380
 381             /* do lookup */
 382             ip4_src_fib_lookup_two (fib_index0, fib_index1,
 383                                     input_addr0, input_addr1,
 384                                     &lbi0, &lbi1);
 385             lb0 = load_balance_get(lbi0);
 386             lb1 = load_balance_get(lbi1);
 387
 388             /* Use flow hash to compute multipath adjacency. */
 389             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 390             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 391
 392             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 393             {
 394                 flow_hash_config0 = lb0->lb_hash_config;
 395                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 396                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 397             }
 398
 399             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 400             {
 401                 flow_hash_config1 = lb1->lb_hash_config;
 402                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 403                     ip4_compute_flow_hash (ip1, flow_hash_config1);
 404             }
 405
 406             dpo0 = load_balance_get_bucket_i(lb0,
 407                                              (hash_c0 &
 408                                               (lb0->lb_n_buckets_minus_1)));
 409             dpo1 = load_balance_get_bucket_i(lb1,
 410                                              (hash_c1 &
 411                                               (lb1->lb_n_buckets_minus_1)));
 412
 413             next0 = dpo0->dpoi_next_node;
 414             next1 = dpo1->dpoi_next_node;
 415             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 416             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 417
 418             vlib_increment_combined_counter
 419                 (cm, cpu_index, lbi0, 1,
 420                  vlib_buffer_length_in_chain (vm, b0));
 421             vlib_increment_combined_counter
 422                 (cm, cpu_index, lbi1, 1,
 423                  vlib_buffer_length_in_chain (vm, b1));
 424
 425             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 426             {
 427                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 428                                                      b0, sizeof (*tr));
 429                 tr->fib_index = fib_index0;
 430                 tr->lbi = lbi0;
 431                 tr->addr.ip4 = *input_addr0;
 432             }
 433             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 434             {
 435                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 436                                                      b1, sizeof (*tr));
 437                 tr->fib_index = fib_index1;
 438                 tr->lbi = lbi1;
 439                 tr->addr.ip4 = *input_addr1;
 440             }
 441
 442             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 443                                              to_next, n_left_to_next,
 444                                              bi0, bi1, next0, next1);
 445         }
 446
 447         while (n_left_from > 0 && n_left_to_next > 0)
 448         {
 449             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 450             flow_hash_config_t flow_hash_config0;
 451             const ip4_address_t *input_addr;
 452             const load_balance_t *lb0;
 453             const lookup_dpo_t * lkd0;
 454             const ip4_header_t * ip0;
 455             const dpo_id_t *dpo0;
 456             vlib_buffer_t * b0;
 457
 458             bi0 = from[0];
 459             to_next[0] = bi0;
 460             from += 1;
 461             to_next += 1;
 462             n_left_from -= 1;
 463             n_left_to_next -= 1;
 464
 465             b0 = vlib_get_buffer (vm, bi0);
 466             ip0 = vlib_buffer_get_current (b0);
 467
 468             /* dst lookup was done by ip4 lookup */
 469             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 470             lkd0 = lookup_dpo_get(lkdi0);
 471
 472             /*
 473              * choose between a lookup using the fib index in the DPO
 474              * or getting the FIB index from the interface.
 475              */
 476             if (table_from_interface)
 477             {
 478                 fib_index0 =
 479                     ip4_fib_table_get_index_for_sw_if_index(
 480                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 481             }
 482             else
 483             {
 484                 fib_index0 = lkd0->lkd_fib_index;
 485             }
 486
 487             /*
 488              * choose between a source or destination address lookup in the table
 489              */
 490             if (input_src_addr)
 491             {
 492                 input_addr = &ip0->src_address;
 493             }
 494             else
 495             {
 496                 input_addr = &ip0->dst_address;
 497             }
 498
 499             /* do lookup */
 500             ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
 501             lb0 = load_balance_get(lbi0);
 502
 503             /* Use flow hash to compute multipath adjacency. */
 504             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 505
 506             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 507             {
 508                 flow_hash_config0 = lb0->lb_hash_config;
 509                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 510                     ip4_compute_flow_hash (ip0, flow_hash_config0);
 511             }
 512
 513             dpo0 = load_balance_get_bucket_i(lb0,
 514                                              (hash_c0 &
 515                                               (lb0->lb_n_buckets_minus_1)));
 516
 517             next0 = dpo0->dpoi_next_node;
 518             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 519
 520             vlib_increment_combined_counter
 521                 (cm, cpu_index, lbi0, 1,
 522                  vlib_buffer_length_in_chain (vm, b0));
 523
 524             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 525             {
 526                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 527                                                      b0, sizeof (*tr));
 528                 tr->fib_index = fib_index0;
 529                 tr->lbi = lbi0;
 530                 tr->addr.ip4 = *input_addr;
 531             }
 532
 533             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 534                                             n_left_to_next, bi0, next0);
 535         }
 536         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 537     }
 538     return from_frame->n_vectors;
 539 }
 540
 541 static u8 *
 542 format_lookup_trace (u8 * s, va_list * args)
 543 {
 544     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 545     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 546     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
 547     uword indent = format_get_indent (s);
 548     s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
 549                 format_white_space, indent,
 550                 t->fib_index,
 551                 format_ip46_address, &t->addr, IP46_TYPE_ANY,
 552                 t->lbi);
 553     return s;
 554 }
 555
 556 always_inline uword
 557 lookup_ip4_dst (vlib_main_t * vm,
 558                 vlib_node_runtime_t * node,
 559                 vlib_frame_t * from_frame)
 560 {
 561     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
 562 }
 563
 564 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
 565     .function = lookup_ip4_dst,
 566     .name = "lookup-ip4-dst",
 567     .vector_size = sizeof (u32),
 568     .sibling_of = "ip4-lookup",
 569     .format_trace = format_lookup_trace,
 570 };
 571 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_node, lookup_ip4_dst)
 572
 573 always_inline uword
 574 lookup_ip4_dst_itf (vlib_main_t * vm,
 575                     vlib_node_runtime_t * node,
 576                     vlib_frame_t * from_frame)
 577 {
 578     return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
 579 }
 580
 581 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
 582     .function = lookup_ip4_dst_itf,
 583     .name = "lookup-ip4-dst-itf",
 584     .vector_size = sizeof (u32),
 585     .sibling_of = "ip4-lookup",
 586     .format_trace = format_lookup_trace,
 587 };
 588 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_itf_node, lookup_ip4_dst_itf)
 589
 590 always_inline uword
 591 lookup_ip4_src (vlib_main_t * vm,
 592                 vlib_node_runtime_t * node,
 593                 vlib_frame_t * from_frame)
 594 {
 595     return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
 596 }
 597
 598 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
 599     .function = lookup_ip4_src,
 600     .name = "lookup-ip4-src",
 601     .vector_size = sizeof (u32),
 602     .format_trace = format_lookup_trace,
 603     .sibling_of = "ip4-lookup",
 604 };
 605 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_src_node, lookup_ip4_src)
 606
 607 always_inline uword
 608 lookup_dpo_ip6_inline (vlib_main_t * vm,
 609                        vlib_node_runtime_t * node,
 610                        vlib_frame_t * from_frame,
 611                        int input_src_addr,
 612                        int table_from_interface)
 613 {
 614     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 615     u32 n_left_from, next_index, * from, * to_next;
 616     u32 cpu_index = os_get_cpu_number();
 617
 618     from = vlib_frame_vector_args (from_frame);
 619     n_left_from = from_frame->n_vectors;
 620
 621     next_index = node->cached_next_index;
 622
 623     while (n_left_from > 0)
 624     {
 625         u32 n_left_to_next;
 626
 627         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 628
 629         while (n_left_from >= 4 && n_left_to_next > 2)
 630         {
 631             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 632             flow_hash_config_t flow_hash_config0;
 633             const ip6_address_t *input_addr0;
 634             const load_balance_t *lb0;
 635             const lookup_dpo_t * lkd0;
 636             const ip6_header_t * ip0;
 637             const dpo_id_t *dpo0;
 638             vlib_buffer_t * b0;
 639             u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
 640             flow_hash_config_t flow_hash_config1;
 641             const ip6_address_t *input_addr1;
 642             const load_balance_t *lb1;
 643             const lookup_dpo_t * lkd1;
 644             const ip6_header_t * ip1;
 645             const dpo_id_t *dpo1;
 646             vlib_buffer_t * b1;
 647
 648             /* Prefetch next iteration. */
 649             {
 650                 vlib_buffer_t * p2, * p3;
 651
 652                 p2 = vlib_get_buffer (vm, from[2]);
 653                 p3 = vlib_get_buffer (vm, from[3]);
 654
 655                 vlib_prefetch_buffer_header (p2, LOAD);
 656                 vlib_prefetch_buffer_header (p3, LOAD);
 657
 658                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 659                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
 660             }
 661
 662             bi0 = from[0];
 663             to_next[0] = bi0;
 664             bi1 = from[1];
 665             to_next[1] = bi1;
 666             from += 2;
 667             to_next += 2;
 668             n_left_from -= 2;
 669             n_left_to_next -= 2;
 670
 671             b0 = vlib_get_buffer (vm, bi0);
 672             ip0 = vlib_buffer_get_current (b0);
 673             b1 = vlib_get_buffer (vm, bi1);
 674             ip1 = vlib_buffer_get_current (b1);
 675
 676             /* dst lookup was done by ip6 lookup */
 677             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 678             lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 679             lkd0 = lookup_dpo_get(lkdi0);
 680             lkd1 = lookup_dpo_get(lkdi1);
 681
 682             /*
 683              * choose between a lookup using the fib index in the DPO
 684              * or getting the FIB index from the interface.
 685              */
 686             if (table_from_interface)
 687             {
 688                 fib_index0 =
 689                     ip6_fib_table_get_index_for_sw_if_index(
 690                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 691                 fib_index1 =
 692                     ip6_fib_table_get_index_for_sw_if_index(
 693                         vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 694             }
 695             else
 696             {
 697                 fib_index0 = lkd0->lkd_fib_index;
 698                 fib_index1 = lkd1->lkd_fib_index;
 699             }
 700
 701             /*
 702              * choose between a source or destination address lookup in the table
 703              */
 704             if (input_src_addr)
 705             {
 706                 input_addr0 = &ip0->src_address;
 707                 input_addr1 = &ip1->src_address;
 708             }
 709             else
 710             {
 711                 input_addr0 = &ip0->dst_address;
 712                 input_addr1 = &ip1->dst_address;
 713             }
 714
 715             /* do src lookup */
 716             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 717                                                fib_index0,
 718                                                input_addr0);
 719             lbi1 = ip6_fib_table_fwding_lookup(&ip6_main,
 720                                                fib_index1,
 721                                                input_addr1);
 722             lb0 = load_balance_get(lbi0);
 723             lb1 = load_balance_get(lbi1);
 724
 725             /* Use flow hash to compute multipath adjacency. */
 726             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 727             hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
 728
 729             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 730             {
 731                 flow_hash_config0 = lb0->lb_hash_config;
 732                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 733                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 734             }
 735
 736             if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 737             {
 738                 flow_hash_config1 = lb1->lb_hash_config;
 739                 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
 740                     ip6_compute_flow_hash (ip1, flow_hash_config1);
 741             }
 742
 743             dpo0 = load_balance_get_bucket_i(lb0,
 744                                              (hash_c0 &
 745                                               (lb0->lb_n_buckets_minus_1)));
 746             dpo1 = load_balance_get_bucket_i(lb1,
 747                                              (hash_c1 &
 748                                               (lb1->lb_n_buckets_minus_1)));
 749
 750             next0 = dpo0->dpoi_next_node;
 751             next1 = dpo1->dpoi_next_node;
 752             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 753             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 754
 755             vlib_increment_combined_counter
 756                 (cm, cpu_index, lbi0, 1,
 757                  vlib_buffer_length_in_chain (vm, b0));
 758             vlib_increment_combined_counter
 759                 (cm, cpu_index, lbi1, 1,
 760                  vlib_buffer_length_in_chain (vm, b1));
 761
 762             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 763             {
 764                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 765                                                      b0, sizeof (*tr));
 766                 tr->fib_index = fib_index0;
 767                 tr->lbi = lbi0;
 768                 tr->addr.ip6 = *input_addr0;
 769             }
 770             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 771             {
 772                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 773                                                      b1, sizeof (*tr));
 774                 tr->fib_index = fib_index1;
 775                 tr->lbi = lbi1;
 776                 tr->addr.ip6 = *input_addr1;
 777             }
 778             vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
 779                                             n_left_to_next, bi0, bi1,
 780                                             next0, next1);
 781         }
 782         while (n_left_from > 0 && n_left_to_next > 0)
 783         {
 784             u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
 785             flow_hash_config_t flow_hash_config0;
 786             const ip6_address_t *input_addr0;
 787             const load_balance_t *lb0;
 788             const lookup_dpo_t * lkd0;
 789             const ip6_header_t * ip0;
 790             const dpo_id_t *dpo0;
 791             vlib_buffer_t * b0;
 792
 793             bi0 = from[0];
 794             to_next[0] = bi0;
 795             from += 1;
 796             to_next += 1;
 797             n_left_from -= 1;
 798             n_left_to_next -= 1;
 799
 800             b0 = vlib_get_buffer (vm, bi0);
 801             ip0 = vlib_buffer_get_current (b0);
 802
 803             /* dst lookup was done by ip6 lookup */
 804             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 805             lkd0 = lookup_dpo_get(lkdi0);
 806
 807             /*
 808              * choose between a lookup using the fib index in the DPO
 809              * or getting the FIB index from the interface.
 810              */
 811             if (table_from_interface)
 812             {
 813                 fib_index0 =
 814                     ip6_fib_table_get_index_for_sw_if_index(
 815                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 816             }
 817             else
 818             {
 819                 fib_index0 = lkd0->lkd_fib_index;
 820             }
 821
 822             /*
 823              * choose between a source or destination address lookup in the table
 824              */
 825             if (input_src_addr)
 826             {
 827                 input_addr0 = &ip0->src_address;
 828             }
 829             else
 830             {
 831                 input_addr0 = &ip0->dst_address;
 832             }
 833
 834             /* do src lookup */
 835             lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
 836                                                fib_index0,
 837                                                input_addr0);
 838             lb0 = load_balance_get(lbi0);
 839
 840             /* Use flow hash to compute multipath adjacency. */
 841             hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
 842
 843             if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 844             {
 845                 flow_hash_config0 = lb0->lb_hash_config;
 846                 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
 847                     ip6_compute_flow_hash (ip0, flow_hash_config0);
 848             }
 849
 850             dpo0 = load_balance_get_bucket_i(lb0,
 851                                              (hash_c0 &
 852                                               (lb0->lb_n_buckets_minus_1)));
 853
 854             next0 = dpo0->dpoi_next_node;
 855             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 856
 857             vlib_increment_combined_counter
 858                 (cm, cpu_index, lbi0, 1,
 859                  vlib_buffer_length_in_chain (vm, b0));
 860
 861             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 862             {
 863                 lookup_trace_t *tr = vlib_add_trace (vm, node,
 864                                                      b0, sizeof (*tr));
 865                 tr->fib_index = fib_index0;
 866                 tr->lbi = lbi0;
 867                 tr->addr.ip6 = *input_addr0;
 868             }
 869             vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
 870                                             n_left_to_next, bi0, next0);
 871         }
 872         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 873     }
 874     return from_frame->n_vectors;
 875 }
 876
 877 always_inline uword
 878 lookup_ip6_dst (vlib_main_t * vm,
 879                 vlib_node_runtime_t * node,
 880                 vlib_frame_t * from_frame)
 881 {
 882     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 0));
 883 }
 884
 885 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
 886     .function = lookup_ip6_dst,
 887     .name = "lookup-ip6-dst",
 888     .vector_size = sizeof (u32),
 889     .format_trace = format_lookup_trace,
 890     .sibling_of = "ip6-lookup",
 891 };
 892 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_node, lookup_ip6_dst)
 893
 894 always_inline uword
 895 lookup_ip6_dst_itf (vlib_main_t * vm,
 896                     vlib_node_runtime_t * node,
 897                     vlib_frame_t * from_frame)
 898 {
 899     return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 1));
 900 }
 901
 902 VLIB_REGISTER_NODE (lookup_ip6_dst_itf_node) = {
 903     .function = lookup_ip6_dst_itf,
 904     .name = "lookup-ip6-dst-itf",
 905     .vector_size = sizeof (u32),
 906     .format_trace = format_lookup_trace,
 907     .sibling_of = "ip6-lookup",
 908 };
 909 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_itf_node, lookup_ip6_dst_itf)
 910
 911 always_inline uword
 912 lookup_ip6_src (vlib_main_t * vm,
 913                 vlib_node_runtime_t * node,
 914                 vlib_frame_t * from_frame)
 915 {
 916     return (lookup_dpo_ip6_inline(vm, node, from_frame, 1, 0));
 917 }
 918
 919 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
 920     .function = lookup_ip6_src,
 921     .name = "lookup-ip6-src",
 922     .vector_size = sizeof (u32),
 923     .format_trace = format_lookup_trace,
 924     .sibling_of = "ip6-lookup",
 925 };
 926 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_src_node, lookup_ip6_src)
 927
 928 always_inline uword
 929 lookup_dpo_mpls_inline (vlib_main_t * vm,
 930                        vlib_node_runtime_t * node,
 931                        vlib_frame_t * from_frame,
 932                        int table_from_interface)
 933 {
 934     u32 n_left_from, next_index, * from, * to_next;
 935     u32 cpu_index = os_get_cpu_number();
 936     vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
 937
 938     from = vlib_frame_vector_args (from_frame);
 939     n_left_from = from_frame->n_vectors;
 940
 941     next_index = node->cached_next_index;
 942
 943     while (n_left_from > 0)
 944     {
 945         u32 n_left_to_next;
 946
 947         vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
 948
 949         /* while (n_left_from >= 4 && n_left_to_next >= 2) */
 950         /*   } */
 951
 952         while (n_left_from > 0 && n_left_to_next > 0)
 953         {
 954             u32 bi0, lkdi0, lbi0, fib_index0,  next0;
 955             const mpls_unicast_header_t * hdr0;
 956             const load_balance_t *lb0;
 957             const lookup_dpo_t * lkd0;
 958             const dpo_id_t *dpo0;
 959             vlib_buffer_t * b0;
 960
 961             bi0 = from[0];
 962             to_next[0] = bi0;
 963             from += 1;
 964             to_next += 1;
 965             n_left_from -= 1;
 966             n_left_to_next -= 1;
 967
 968             b0 = vlib_get_buffer (vm, bi0);
 969             hdr0 = vlib_buffer_get_current (b0);
 970
 971             /* dst lookup was done by mpls lookup */
 972             lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 973             lkd0 = lookup_dpo_get(lkdi0);
 974
 975             /*
 976              * choose between a lookup using the fib index in the DPO
 977              * or getting the FIB index from the interface.
 978              */
 979             if (table_from_interface)
 980             {
 981                 fib_index0 =
 982                     mpls_fib_table_get_index_for_sw_if_index(
 983                         vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 984             }
 985             else
 986             {
 987                 fib_index0 = lkd0->lkd_fib_index;
 988             }
 989
 990             /* do lookup */
 991             lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
 992             lb0  = load_balance_get(lbi0);
 993             dpo0 = load_balance_get_bucket_i(lb0, 0);
 994
 995             next0 = dpo0->dpoi_next_node;
 996             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 997
 998             vlib_increment_combined_counter
 999                 (cm, cpu_index, lbi0, 1,
1000                  vlib_buffer_length_in_chain (vm, b0));
1001
1002             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1003             {
1004                 lookup_trace_t *tr = vlib_add_trace (vm, node,
1005                                                      b0, sizeof (*tr));
1006                 tr->fib_index = fib_index0;
1007                 tr->lbi = lbi0;
1008                 tr->hdr = *hdr0;
1009             }
1010
1011            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1012                                             n_left_to_next, bi0, next0);
1013         }
1014         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1015     }
1016     return from_frame->n_vectors;
1017 }
1018
1019 static u8 *
1020 format_lookup_mpls_trace (u8 * s, va_list * args)
1021 {
1022     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1023     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1024     lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
1025     uword indent = format_get_indent (s);
1026     mpls_unicast_header_t hdr;
1027
1028     hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
1029
1030     s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
1031                 format_white_space, indent,
1032                 t->fib_index,
1033                 format_mpls_header, hdr,
1034                 t->lbi);
1035     return s;
1036 }
1037
1038 always_inline uword
1039 lookup_mpls_dst (vlib_main_t * vm,
1040                 vlib_node_runtime_t * node,
1041                 vlib_frame_t * from_frame)
1042 {
1043     return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
1044 }
1045
1046 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
1047     .function = lookup_mpls_dst,
1048     .name = "lookup-mpls-dst",
1049     .vector_size = sizeof (u32),
1050     .sibling_of = "mpls-lookup",
1051     .format_trace = format_lookup_mpls_trace,
1052     .n_next_nodes = 0,
1053 };
1054 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_node, lookup_mpls_dst)
1055
1056 always_inline uword
1057 lookup_mpls_dst_itf (vlib_main_t * vm,
1058                     vlib_node_runtime_t * node,
1059                     vlib_frame_t * from_frame)
1060 {
1061     return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
1062 }
1063
1064 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
1065     .function = lookup_mpls_dst_itf,
1066     .name = "lookup-mpls-dst-itf",
1067     .vector_size = sizeof (u32),
1068     .sibling_of = "mpls-lookup",
1069     .format_trace = format_lookup_mpls_trace,
1070     .n_next_nodes = 0,
1071 };
1072 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf)
1073
1074 static void
1075 lookup_dpo_mem_show (void)
1076 {
1077     fib_show_memory_usage("Lookup",
1078                           pool_elts(lookup_dpo_pool),
1079                           pool_len(lookup_dpo_pool),
1080                           sizeof(lookup_dpo_t));
1081 }
1082
1083 const static dpo_vft_t lkd_vft = {
1084     .dv_lock = lookup_dpo_lock,
1085     .dv_unlock = lookup_dpo_unlock,
1086     .dv_format = format_lookup_dpo,
1087 };
1088 const static dpo_vft_t lkd_vft_w_mem_show = {
1089     .dv_lock = lookup_dpo_lock,
1090     .dv_unlock = lookup_dpo_unlock,
1091     .dv_format = format_lookup_dpo,
1092     .dv_mem_show = lookup_dpo_mem_show,
1093 };
1094
1095 const static char* const lookup_src_ip4_nodes[] =
1096 {
1097     "lookup-ip4-src",
1098     NULL,
1099 };
1100 const static char* const lookup_src_ip6_nodes[] =
1101 {
1102     "lookup-ip6-src",
1103     NULL,
1104 };
1105 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
1106 {
1107     [DPO_PROTO_IP4]  = lookup_src_ip4_nodes,
1108     [DPO_PROTO_IP6]  = lookup_src_ip6_nodes,
1109     [DPO_PROTO_MPLS] = NULL,
1110 };
1111
1112 const static char* const lookup_dst_ip4_nodes[] =
1113 {
1114     "lookup-ip4-dst",
1115     NULL,
1116 };
1117 const static char* const lookup_dst_ip6_nodes[] =
1118 {
1119     "lookup-ip6-dst",
1120     NULL,
1121 };
1122 const static char* const lookup_dst_mpls_nodes[] =
1123 {
1124     "lookup-mpls-dst",
1125     NULL,
1126 };
1127 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
1128 {
1129     [DPO_PROTO_IP4]  = lookup_dst_ip4_nodes,
1130     [DPO_PROTO_IP6]  = lookup_dst_ip6_nodes,
1131     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
1132 };
1133
1134 const static char* const lookup_dst_from_interface_ip4_nodes[] =
1135 {
1136     "lookup-ip4-dst-itf",
1137     NULL,
1138 };
1139 const static char* const lookup_dst_from_interface_ip6_nodes[] =
1140 {
1141     "lookup-ip6-dst-itf",
1142     NULL,
1143 };
1144 const static char* const lookup_dst_from_interface_mpls_nodes[] =
1145 {
1146     "lookup-mpls-dst-itf",
1147     NULL,
1148 };
1149 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
1150 {
1151     [DPO_PROTO_IP4]  = lookup_dst_from_interface_ip4_nodes,
1152     [DPO_PROTO_IP6]  = lookup_dst_from_interface_ip6_nodes,
1153     [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
1154 };
1155
1156
1157 void
1158 lookup_dpo_module_init (void)
1159 {
1160     dpo_register(DPO_LOOKUP, &lkd_vft_w_mem_show, NULL);
1161
1162     /*
1163      * There are various sorts of lookup; src or dst addr v4 /v6 etc.
1164      * there isn't an object type for each (there is only the lookup_dpo_t),
1165      * but, for performance reasons, there is a data plane function, and hence
1166      * VLIB node for each. VLIB graph node construction is based on DPO types
1167      * so we create sub-types.
1168      */
1169     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
1170         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
1171     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
1172         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
1173     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
1174         dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes);
1175 }