2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/ip/ip.h>
17 #include <vnet/dpo/lookup_dpo.h>
18 #include <vnet/dpo/load_balance.h>
19 #include <vnet/mpls/mpls.h>
20 #include <vnet/fib/fib_table.h>
21 #include <vnet/fib/ip4_fib.h>
22 #include <vnet/fib/ip6_fib.h>
23 #include <vnet/fib/mpls_fib.h>
24 #include <vnet/mfib/mfib_table.h>
25 #include <vnet/mfib/ip4_mfib.h>
26 #include <vnet/mfib/ip6_mfib.h>
28 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
29 static const char *const lookup_cast_names[] = LOOKUP_CASTS;
32 * @brief Enumeration of the lookup subtypes
34 typedef enum lookup_sub_type_t_
38 LOOKUP_SUB_TYPE_DST_MCAST,
39 LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
41 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
43 #define FOR_EACH_LOOKUP_SUB_TYPE(_st) \
44 for (_st = LOOKUP_SUB_TYPE_IP4_SRC; _st < LOOKUP_SUB_TYPE_NUM; _st++)
47 * @brief pool of all MPLS Label DPOs
49 lookup_dpo_t *lookup_dpo_pool;
52 * @brief An array of registered DPO type values for the sub-types
54 static dpo_type_t lookup_dpo_sub_types[LOOKUP_SUB_TYPE_NUM];
57 lookup_dpo_alloc (void)
61 pool_get_aligned(lookup_dpo_pool, lkd, CLIB_CACHE_LINE_BYTES);
67 lookup_dpo_get_index (lookup_dpo_t *lkd)
69 return (lkd - lookup_dpo_pool);
73 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
77 lookup_table_t table_config,
83 lkd = lookup_dpo_alloc();
84 lkd->lkd_fib_index = fib_index;
85 lkd->lkd_proto = proto;
86 lkd->lkd_input = input;
87 lkd->lkd_table = table_config;
91 * use the input type to select the lookup sub-type
97 case LOOKUP_INPUT_SRC_ADDR:
98 type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC];
100 case LOOKUP_INPUT_DST_ADDR:
101 switch (table_config)
103 case LOOKUP_TABLE_FROM_INPUT_INTERFACE:
104 type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE];
106 case LOOKUP_TABLE_FROM_CONFIG:
107 type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
110 if (LOOKUP_MULTICAST == cast)
112 type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST];
122 dpo_set(dpo, type, proto, lookup_dpo_get_index(lkd));
127 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
130 lookup_input_t input,
131 lookup_table_t table_config,
134 if (LOOKUP_TABLE_FROM_CONFIG == table_config)
136 if (LOOKUP_UNICAST == cast)
138 fib_table_lock(fib_index, dpo_proto_to_fib(proto));
142 mfib_table_lock(fib_index, dpo_proto_to_fib(proto));
145 lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
149 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
152 lookup_input_t input,
153 lookup_table_t table_config,
156 fib_node_index_t fib_index = FIB_NODE_INDEX_INVALID;
158 if (LOOKUP_TABLE_FROM_CONFIG == table_config)
160 if (LOOKUP_UNICAST == cast)
163 fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
169 mfib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
174 ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
175 lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
179 format_lookup_dpo (u8 *s, va_list *args)
181 index_t index = va_arg (*args, index_t);
184 lkd = lookup_dpo_get(index);
186 if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
188 s = format(s, "%s,%s lookup in interface's %U table",
189 lookup_input_names[lkd->lkd_input],
190 lookup_cast_names[lkd->lkd_cast],
191 format_dpo_proto, lkd->lkd_proto);
195 if (LOOKUP_UNICAST == lkd->lkd_cast)
197 s = format(s, "%s,%s lookup in %U",
198 lookup_input_names[lkd->lkd_input],
199 lookup_cast_names[lkd->lkd_cast],
200 format_fib_table_name, lkd->lkd_fib_index,
201 dpo_proto_to_fib(lkd->lkd_proto));
205 s = format(s, "%s,%s lookup in %U",
206 lookup_input_names[lkd->lkd_input],
207 lookup_cast_names[lkd->lkd_cast],
208 format_mfib_table_name, lkd->lkd_fib_index,
209 dpo_proto_to_fib(lkd->lkd_proto));
216 lookup_dpo_lock (dpo_id_t *dpo)
220 lkd = lookup_dpo_get(dpo->dpoi_index);
226 lookup_dpo_unlock (dpo_id_t *dpo)
230 lkd = lookup_dpo_get(dpo->dpoi_index);
234 if (0 == lkd->lkd_locks)
236 if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
238 if (LOOKUP_UNICAST == lkd->lkd_cast)
240 fib_table_unlock(lkd->lkd_fib_index,
241 dpo_proto_to_fib(lkd->lkd_proto));
245 mfib_table_unlock(lkd->lkd_fib_index,
246 dpo_proto_to_fib(lkd->lkd_proto));
249 pool_put(lookup_dpo_pool, lkd);
254 ip4_src_fib_lookup_one (u32 src_fib_index0,
255 const ip4_address_t * addr0,
256 u32 * src_adj_index0)
258 ip4_fib_mtrie_leaf_t leaf0;
259 ip4_fib_mtrie_t * mtrie0;
261 mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
263 leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
264 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
265 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
267 src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
271 ip4_src_fib_lookup_two (u32 src_fib_index0,
273 const ip4_address_t * addr0,
274 const ip4_address_t * addr1,
275 u32 * src_adj_index0,
276 u32 * src_adj_index1)
278 ip4_fib_mtrie_leaf_t leaf0, leaf1;
279 ip4_fib_mtrie_t * mtrie0, * mtrie1;
281 mtrie0 = &ip4_fib_get (src_fib_index0)->mtrie;
282 mtrie1 = &ip4_fib_get (src_fib_index1)->mtrie;
284 leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, addr0);
285 leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, addr1);
287 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
288 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
290 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
291 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
293 src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
294 src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
298 * @brief Lookup trace data
300 typedef struct lookup_trace_t_
304 mpls_unicast_header_t hdr;
306 fib_node_index_t fib_index;
312 lookup_dpo_ip4_inline (vlib_main_t * vm,
313 vlib_node_runtime_t * node,
314 vlib_frame_t * from_frame,
316 int table_from_interface)
318 u32 n_left_from, next_index, * from, * to_next;
319 u32 thread_index = vlib_get_thread_index();
320 vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
322 from = vlib_frame_vector_args (from_frame);
323 n_left_from = from_frame->n_vectors;
325 next_index = node->cached_next_index;
327 while (n_left_from > 0)
331 vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
333 while (n_left_from >= 4 && n_left_to_next > 2)
335 u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
336 flow_hash_config_t flow_hash_config0;
337 const ip4_address_t *input_addr0;
338 const load_balance_t *lb0;
339 const lookup_dpo_t * lkd0;
340 const ip4_header_t * ip0;
341 const dpo_id_t *dpo0;
343 u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
344 flow_hash_config_t flow_hash_config1;
345 const ip4_address_t *input_addr1;
346 const load_balance_t *lb1;
347 const lookup_dpo_t * lkd1;
348 const ip4_header_t * ip1;
349 const dpo_id_t *dpo1;
352 /* Prefetch next iteration. */
354 vlib_buffer_t * p2, * p3;
356 p2 = vlib_get_buffer (vm, from[2]);
357 p3 = vlib_get_buffer (vm, from[3]);
359 vlib_prefetch_buffer_header (p2, LOAD);
360 vlib_prefetch_buffer_header (p3, LOAD);
362 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
363 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
375 b0 = vlib_get_buffer (vm, bi0);
376 ip0 = vlib_buffer_get_current (b0);
377 b1 = vlib_get_buffer (vm, bi1);
378 ip1 = vlib_buffer_get_current (b1);
380 /* dst lookup was done by ip4 lookup */
381 lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
382 lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
383 lkd0 = lookup_dpo_get(lkdi0);
384 lkd1 = lookup_dpo_get(lkdi1);
387 * choose between a lookup using the fib index in the DPO
388 * or getting the FIB index from the interface.
390 if (table_from_interface)
393 ip4_fib_table_get_index_for_sw_if_index(
394 vnet_buffer(b0)->sw_if_index[VLIB_RX]);
396 ip4_fib_table_get_index_for_sw_if_index(
397 vnet_buffer(b1)->sw_if_index[VLIB_RX]);
401 fib_index0 = lkd0->lkd_fib_index;
402 fib_index1 = lkd1->lkd_fib_index;
406 * choose between a source or destination address lookup in the table
410 input_addr0 = &ip0->src_address;
411 input_addr1 = &ip1->src_address;
415 input_addr0 = &ip0->dst_address;
416 input_addr1 = &ip1->dst_address;
420 ip4_src_fib_lookup_two (fib_index0, fib_index1,
421 input_addr0, input_addr1,
423 lb0 = load_balance_get(lbi0);
424 lb1 = load_balance_get(lbi1);
426 vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
427 vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
429 /* Use flow hash to compute multipath adjacency. */
430 hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
431 hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
433 if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
435 flow_hash_config0 = lb0->lb_hash_config;
436 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
437 ip4_compute_flow_hash (ip0, flow_hash_config0);
440 if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
442 flow_hash_config1 = lb1->lb_hash_config;
443 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
444 ip4_compute_flow_hash (ip1, flow_hash_config1);
447 dpo0 = load_balance_get_bucket_i(lb0,
449 (lb0->lb_n_buckets_minus_1)));
450 dpo1 = load_balance_get_bucket_i(lb1,
452 (lb1->lb_n_buckets_minus_1)));
454 next0 = dpo0->dpoi_next_node;
455 next1 = dpo1->dpoi_next_node;
456 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
457 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
459 vlib_increment_combined_counter
460 (cm, thread_index, lbi0, 1,
461 vlib_buffer_length_in_chain (vm, b0));
462 vlib_increment_combined_counter
463 (cm, thread_index, lbi1, 1,
464 vlib_buffer_length_in_chain (vm, b1));
466 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
468 lookup_trace_t *tr = vlib_add_trace (vm, node,
470 tr->fib_index = fib_index0;
472 tr->addr.ip4 = *input_addr0;
474 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
476 lookup_trace_t *tr = vlib_add_trace (vm, node,
478 tr->fib_index = fib_index1;
480 tr->addr.ip4 = *input_addr1;
483 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
484 to_next, n_left_to_next,
485 bi0, bi1, next0, next1);
488 while (n_left_from > 0 && n_left_to_next > 0)
490 u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
491 flow_hash_config_t flow_hash_config0;
492 const ip4_address_t *input_addr;
493 const load_balance_t *lb0;
494 const lookup_dpo_t * lkd0;
495 const ip4_header_t * ip0;
496 const dpo_id_t *dpo0;
506 b0 = vlib_get_buffer (vm, bi0);
507 ip0 = vlib_buffer_get_current (b0);
509 /* dst lookup was done by ip4 lookup */
510 lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
511 lkd0 = lookup_dpo_get(lkdi0);
514 * choose between a lookup using the fib index in the DPO
515 * or getting the FIB index from the interface.
517 if (table_from_interface)
520 ip4_fib_table_get_index_for_sw_if_index(
521 vnet_buffer(b0)->sw_if_index[VLIB_RX]);
525 fib_index0 = lkd0->lkd_fib_index;
529 * choose between a source or destination address lookup in the table
533 input_addr = &ip0->src_address;
537 input_addr = &ip0->dst_address;
541 ip4_src_fib_lookup_one (fib_index0, input_addr, &lbi0);
542 lb0 = load_balance_get(lbi0);
544 vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
546 /* Use flow hash to compute multipath adjacency. */
547 hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
549 if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
551 flow_hash_config0 = lb0->lb_hash_config;
552 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
553 ip4_compute_flow_hash (ip0, flow_hash_config0);
556 dpo0 = load_balance_get_bucket_i(lb0,
558 (lb0->lb_n_buckets_minus_1)));
560 next0 = dpo0->dpoi_next_node;
561 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
563 vlib_increment_combined_counter
564 (cm, thread_index, lbi0, 1,
565 vlib_buffer_length_in_chain (vm, b0));
567 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
569 lookup_trace_t *tr = vlib_add_trace (vm, node,
571 tr->fib_index = fib_index0;
573 tr->addr.ip4 = *input_addr;
576 vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
577 n_left_to_next, bi0, next0);
579 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
581 return from_frame->n_vectors;
585 format_lookup_trace (u8 * s, va_list * args)
587 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
588 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
589 lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
590 uword indent = format_get_indent (s);
591 s = format (s, "%U fib-index:%d addr:%U load-balance:%d",
592 format_white_space, indent,
594 format_ip46_address, &t->addr, IP46_TYPE_ANY,
600 lookup_ip4_dst (vlib_main_t * vm,
601 vlib_node_runtime_t * node,
602 vlib_frame_t * from_frame)
604 return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 0));
607 VLIB_REGISTER_NODE (lookup_ip4_dst_node) = {
608 .function = lookup_ip4_dst,
609 .name = "lookup-ip4-dst",
610 .vector_size = sizeof (u32),
611 .sibling_of = "ip4-lookup",
612 .format_trace = format_lookup_trace,
614 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_node, lookup_ip4_dst)
617 lookup_ip4_dst_itf (vlib_main_t * vm,
618 vlib_node_runtime_t * node,
619 vlib_frame_t * from_frame)
621 return (lookup_dpo_ip4_inline(vm, node, from_frame, 0, 1));
624 VLIB_REGISTER_NODE (lookup_ip4_dst_itf_node) = {
625 .function = lookup_ip4_dst_itf,
626 .name = "lookup-ip4-dst-itf",
627 .vector_size = sizeof (u32),
628 .sibling_of = "ip4-lookup",
629 .format_trace = format_lookup_trace,
631 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_itf_node, lookup_ip4_dst_itf)
634 lookup_ip4_src (vlib_main_t * vm,
635 vlib_node_runtime_t * node,
636 vlib_frame_t * from_frame)
638 return (lookup_dpo_ip4_inline(vm, node, from_frame, 1, 0));
641 VLIB_REGISTER_NODE (lookup_ip4_src_node) = {
642 .function = lookup_ip4_src,
643 .name = "lookup-ip4-src",
644 .vector_size = sizeof (u32),
645 .format_trace = format_lookup_trace,
646 .sibling_of = "ip4-lookup",
648 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_src_node, lookup_ip4_src)
651 lookup_dpo_ip6_inline (vlib_main_t * vm,
652 vlib_node_runtime_t * node,
653 vlib_frame_t * from_frame,
655 int table_from_interface)
657 vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
658 u32 n_left_from, next_index, * from, * to_next;
659 u32 thread_index = vlib_get_thread_index();
661 from = vlib_frame_vector_args (from_frame);
662 n_left_from = from_frame->n_vectors;
664 next_index = node->cached_next_index;
666 while (n_left_from > 0)
670 vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
672 while (n_left_from >= 4 && n_left_to_next > 2)
674 u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
675 flow_hash_config_t flow_hash_config0;
676 const ip6_address_t *input_addr0;
677 const load_balance_t *lb0;
678 const lookup_dpo_t * lkd0;
679 const ip6_header_t * ip0;
680 const dpo_id_t *dpo0;
682 u32 bi1, lkdi1, lbi1, fib_index1, next1, hash_c1;
683 flow_hash_config_t flow_hash_config1;
684 const ip6_address_t *input_addr1;
685 const load_balance_t *lb1;
686 const lookup_dpo_t * lkd1;
687 const ip6_header_t * ip1;
688 const dpo_id_t *dpo1;
691 /* Prefetch next iteration. */
693 vlib_buffer_t * p2, * p3;
695 p2 = vlib_get_buffer (vm, from[2]);
696 p3 = vlib_get_buffer (vm, from[3]);
698 vlib_prefetch_buffer_header (p2, LOAD);
699 vlib_prefetch_buffer_header (p3, LOAD);
701 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
702 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
714 b0 = vlib_get_buffer (vm, bi0);
715 ip0 = vlib_buffer_get_current (b0);
716 b1 = vlib_get_buffer (vm, bi1);
717 ip1 = vlib_buffer_get_current (b1);
719 /* dst lookup was done by ip6 lookup */
720 lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
721 lkdi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
722 lkd0 = lookup_dpo_get(lkdi0);
723 lkd1 = lookup_dpo_get(lkdi1);
726 * choose between a lookup using the fib index in the DPO
727 * or getting the FIB index from the interface.
729 if (table_from_interface)
732 ip6_fib_table_get_index_for_sw_if_index(
733 vnet_buffer(b0)->sw_if_index[VLIB_RX]);
735 ip6_fib_table_get_index_for_sw_if_index(
736 vnet_buffer(b1)->sw_if_index[VLIB_RX]);
740 fib_index0 = lkd0->lkd_fib_index;
741 fib_index1 = lkd1->lkd_fib_index;
745 * choose between a source or destination address lookup in the table
749 input_addr0 = &ip0->src_address;
750 input_addr1 = &ip1->src_address;
754 input_addr0 = &ip0->dst_address;
755 input_addr1 = &ip1->dst_address;
759 lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
762 lbi1 = ip6_fib_table_fwding_lookup(&ip6_main,
765 lb0 = load_balance_get(lbi0);
766 lb1 = load_balance_get(lbi1);
768 vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
769 vnet_buffer(b1)->sw_if_index[VLIB_TX] = fib_index1;
771 /* Use flow hash to compute multipath adjacency. */
772 hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
773 hash_c1 = vnet_buffer (b1)->ip.flow_hash = 0;
775 if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
777 flow_hash_config0 = lb0->lb_hash_config;
778 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
779 ip6_compute_flow_hash (ip0, flow_hash_config0);
782 if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
784 flow_hash_config1 = lb1->lb_hash_config;
785 hash_c1 = vnet_buffer (b1)->ip.flow_hash =
786 ip6_compute_flow_hash (ip1, flow_hash_config1);
789 dpo0 = load_balance_get_bucket_i(lb0,
791 (lb0->lb_n_buckets_minus_1)));
792 dpo1 = load_balance_get_bucket_i(lb1,
794 (lb1->lb_n_buckets_minus_1)));
796 next0 = dpo0->dpoi_next_node;
797 next1 = dpo1->dpoi_next_node;
798 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
799 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
801 vlib_increment_combined_counter
802 (cm, thread_index, lbi0, 1,
803 vlib_buffer_length_in_chain (vm, b0));
804 vlib_increment_combined_counter
805 (cm, thread_index, lbi1, 1,
806 vlib_buffer_length_in_chain (vm, b1));
808 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
810 lookup_trace_t *tr = vlib_add_trace (vm, node,
812 tr->fib_index = fib_index0;
814 tr->addr.ip6 = *input_addr0;
816 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
818 lookup_trace_t *tr = vlib_add_trace (vm, node,
820 tr->fib_index = fib_index1;
822 tr->addr.ip6 = *input_addr1;
824 vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
825 n_left_to_next, bi0, bi1,
828 while (n_left_from > 0 && n_left_to_next > 0)
830 u32 bi0, lkdi0, lbi0, fib_index0, next0, hash_c0;
831 flow_hash_config_t flow_hash_config0;
832 const ip6_address_t *input_addr0;
833 const load_balance_t *lb0;
834 const lookup_dpo_t * lkd0;
835 const ip6_header_t * ip0;
836 const dpo_id_t *dpo0;
846 b0 = vlib_get_buffer (vm, bi0);
847 ip0 = vlib_buffer_get_current (b0);
849 /* dst lookup was done by ip6 lookup */
850 lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
851 lkd0 = lookup_dpo_get(lkdi0);
854 * choose between a lookup using the fib index in the DPO
855 * or getting the FIB index from the interface.
857 if (table_from_interface)
860 ip6_fib_table_get_index_for_sw_if_index(
861 vnet_buffer(b0)->sw_if_index[VLIB_RX]);
865 fib_index0 = lkd0->lkd_fib_index;
869 * choose between a source or destination address lookup in the table
873 input_addr0 = &ip0->src_address;
877 input_addr0 = &ip0->dst_address;
881 lbi0 = ip6_fib_table_fwding_lookup(&ip6_main,
884 lb0 = load_balance_get(lbi0);
886 vnet_buffer(b0)->sw_if_index[VLIB_TX] = fib_index0;
888 /* Use flow hash to compute multipath adjacency. */
889 hash_c0 = vnet_buffer (b0)->ip.flow_hash = 0;
891 if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
893 flow_hash_config0 = lb0->lb_hash_config;
894 hash_c0 = vnet_buffer (b0)->ip.flow_hash =
895 ip6_compute_flow_hash (ip0, flow_hash_config0);
898 dpo0 = load_balance_get_bucket_i(lb0,
900 (lb0->lb_n_buckets_minus_1)));
902 next0 = dpo0->dpoi_next_node;
903 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
905 vlib_increment_combined_counter
906 (cm, thread_index, lbi0, 1,
907 vlib_buffer_length_in_chain (vm, b0));
909 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
911 lookup_trace_t *tr = vlib_add_trace (vm, node,
913 tr->fib_index = fib_index0;
915 tr->addr.ip6 = *input_addr0;
917 vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
918 n_left_to_next, bi0, next0);
920 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
922 return from_frame->n_vectors;
926 lookup_ip6_dst (vlib_main_t * vm,
927 vlib_node_runtime_t * node,
928 vlib_frame_t * from_frame)
930 return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 0));
933 VLIB_REGISTER_NODE (lookup_ip6_dst_node) = {
934 .function = lookup_ip6_dst,
935 .name = "lookup-ip6-dst",
936 .vector_size = sizeof (u32),
937 .format_trace = format_lookup_trace,
938 .sibling_of = "ip6-lookup",
940 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_node, lookup_ip6_dst)
943 lookup_ip6_dst_itf (vlib_main_t * vm,
944 vlib_node_runtime_t * node,
945 vlib_frame_t * from_frame)
947 return (lookup_dpo_ip6_inline(vm, node, from_frame, 0 /*use src*/, 1));
950 VLIB_REGISTER_NODE (lookup_ip6_dst_itf_node) = {
951 .function = lookup_ip6_dst_itf,
952 .name = "lookup-ip6-dst-itf",
953 .vector_size = sizeof (u32),
954 .format_trace = format_lookup_trace,
955 .sibling_of = "ip6-lookup",
957 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_dst_itf_node, lookup_ip6_dst_itf)
960 lookup_ip6_src (vlib_main_t * vm,
961 vlib_node_runtime_t * node,
962 vlib_frame_t * from_frame)
964 return (lookup_dpo_ip6_inline(vm, node, from_frame, 1, 0));
967 VLIB_REGISTER_NODE (lookup_ip6_src_node) = {
968 .function = lookup_ip6_src,
969 .name = "lookup-ip6-src",
970 .vector_size = sizeof (u32),
971 .format_trace = format_lookup_trace,
972 .sibling_of = "ip6-lookup",
974 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip6_src_node, lookup_ip6_src)
977 lookup_dpo_mpls_inline (vlib_main_t * vm,
978 vlib_node_runtime_t * node,
979 vlib_frame_t * from_frame,
980 int table_from_interface)
982 u32 n_left_from, next_index, * from, * to_next;
983 u32 thread_index = vlib_get_thread_index();
984 vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
986 from = vlib_frame_vector_args (from_frame);
987 n_left_from = from_frame->n_vectors;
989 next_index = node->cached_next_index;
991 while (n_left_from > 0)
995 vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
997 /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1000 while (n_left_from > 0 && n_left_to_next > 0)
1002 u32 bi0, lkdi0, lbi0, fib_index0, next0;
1003 const mpls_unicast_header_t * hdr0;
1004 const load_balance_t *lb0;
1005 const lookup_dpo_t * lkd0;
1006 const dpo_id_t *dpo0;
1014 n_left_to_next -= 1;
1016 b0 = vlib_get_buffer (vm, bi0);
1017 hdr0 = vlib_buffer_get_current (b0);
1019 /* dst lookup was done by mpls lookup */
1020 lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1021 lkd0 = lookup_dpo_get(lkdi0);
1024 * choose between a lookup using the fib index in the DPO
1025 * or getting the FIB index from the interface.
1027 if (table_from_interface)
1030 mpls_fib_table_get_index_for_sw_if_index(
1031 vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1035 fib_index0 = lkd0->lkd_fib_index;
1039 lbi0 = mpls_fib_table_forwarding_lookup (fib_index0, hdr0);
1040 lb0 = load_balance_get(lbi0);
1041 dpo0 = load_balance_get_bucket_i(lb0, 0);
1043 next0 = dpo0->dpoi_next_node;
1044 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
1046 vlib_increment_combined_counter
1047 (cm, thread_index, lbi0, 1,
1048 vlib_buffer_length_in_chain (vm, b0));
1050 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1052 lookup_trace_t *tr = vlib_add_trace (vm, node,
1054 tr->fib_index = fib_index0;
1059 vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1060 n_left_to_next, bi0, next0);
1062 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1064 return from_frame->n_vectors;
1068 format_lookup_mpls_trace (u8 * s, va_list * args)
1070 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1071 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1072 lookup_trace_t * t = va_arg (*args, lookup_trace_t *);
1073 uword indent = format_get_indent (s);
1074 mpls_unicast_header_t hdr;
1076 hdr.label_exp_s_ttl = clib_net_to_host_u32(t->hdr.label_exp_s_ttl);
1078 s = format (s, "%U fib-index:%d hdr:%U load-balance:%d",
1079 format_white_space, indent,
1081 format_mpls_header, hdr,
1087 lookup_mpls_dst (vlib_main_t * vm,
1088 vlib_node_runtime_t * node,
1089 vlib_frame_t * from_frame)
1091 return (lookup_dpo_mpls_inline(vm, node, from_frame, 0));
1094 VLIB_REGISTER_NODE (lookup_mpls_dst_node) = {
1095 .function = lookup_mpls_dst,
1096 .name = "lookup-mpls-dst",
1097 .vector_size = sizeof (u32),
1098 .sibling_of = "mpls-lookup",
1099 .format_trace = format_lookup_mpls_trace,
1102 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_node, lookup_mpls_dst)
1105 lookup_mpls_dst_itf (vlib_main_t * vm,
1106 vlib_node_runtime_t * node,
1107 vlib_frame_t * from_frame)
1109 return (lookup_dpo_mpls_inline(vm, node, from_frame, 1));
1112 VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
1113 .function = lookup_mpls_dst_itf,
1114 .name = "lookup-mpls-dst-itf",
1115 .vector_size = sizeof (u32),
1116 .sibling_of = "mpls-lookup",
1117 .format_trace = format_lookup_mpls_trace,
1120 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf)
1122 typedef enum lookup_ip_dst_mcast_next_t_ {
1123 LOOKUP_IP_DST_MCAST_NEXT_RPF,
1124 LOOKUP_IP_DST_MCAST_N_NEXT,
1125 } mfib_forward_lookup_next_t;
1128 lookup_dpo_ip_dst_mcast_inline (vlib_main_t * vm,
1129 vlib_node_runtime_t * node,
1130 vlib_frame_t * from_frame,
1133 u32 n_left_from, next_index, * from, * to_next;
1135 from = vlib_frame_vector_args (from_frame);
1136 n_left_from = from_frame->n_vectors;
1138 next_index = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1140 while (n_left_from > 0)
1144 vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
1146 /* while (n_left_from >= 4 && n_left_to_next >= 2) */
1149 while (n_left_from > 0 && n_left_to_next > 0)
1151 u32 bi0, lkdi0, fib_index0, next0;
1152 const lookup_dpo_t * lkd0;
1153 fib_node_index_t mfei0;
1161 n_left_to_next -= 1;
1163 b0 = vlib_get_buffer (vm, bi0);
1165 /* dst lookup was done by mpls lookup */
1166 lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
1167 lkd0 = lookup_dpo_get(lkdi0);
1168 fib_index0 = lkd0->lkd_fib_index;
1169 next0 = LOOKUP_IP_DST_MCAST_NEXT_RPF;
1175 ip0 = vlib_buffer_get_current (b0);
1176 mfei0 = ip4_mfib_table_lookup(ip4_mfib_get(fib_index0),
1180 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1182 lookup_trace_t *tr = vlib_add_trace (vm, node,
1184 tr->fib_index = fib_index0;
1186 tr->addr.ip4 = ip0->dst_address;
1193 ip0 = vlib_buffer_get_current (b0);
1194 mfei0 = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index0),
1197 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
1199 lookup_trace_t *tr = vlib_add_trace (vm, node,
1201 tr->fib_index = fib_index0;
1203 tr->addr.ip6 = ip0->dst_address;
1207 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = mfei0;
1209 vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
1210 n_left_to_next, bi0, next0);
1212 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1214 return from_frame->n_vectors;
1218 lookup_ip4_dst_mcast (vlib_main_t * vm,
1219 vlib_node_runtime_t * node,
1220 vlib_frame_t * from_frame)
1222 return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 1));
1225 VLIB_REGISTER_NODE (lookup_ip4_dst_mcast_node) = {
1226 .function = lookup_ip4_dst_mcast,
1227 .name = "lookup-ip4-dst-mcast",
1228 .vector_size = sizeof (u32),
1230 .format_trace = format_lookup_trace,
1231 .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
1233 [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip4-mfib-forward-rpf",
1236 VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_mcast_node,
1237 lookup_ip4_dst_mcast)
1240 lookup_dpo_mem_show (void)
1242 fib_show_memory_usage("Lookup",
1243 pool_elts(lookup_dpo_pool),
1244 pool_len(lookup_dpo_pool),
1245 sizeof(lookup_dpo_t));
1248 const static dpo_vft_t lkd_vft = {
1249 .dv_lock = lookup_dpo_lock,
1250 .dv_unlock = lookup_dpo_unlock,
1251 .dv_format = format_lookup_dpo,
1253 const static dpo_vft_t lkd_vft_w_mem_show = {
1254 .dv_lock = lookup_dpo_lock,
1255 .dv_unlock = lookup_dpo_unlock,
1256 .dv_format = format_lookup_dpo,
1257 .dv_mem_show = lookup_dpo_mem_show,
1260 const static char* const lookup_src_ip4_nodes[] =
1265 const static char* const lookup_src_ip6_nodes[] =
1270 const static char* const * const lookup_src_nodes[DPO_PROTO_NUM] =
1272 [DPO_PROTO_IP4] = lookup_src_ip4_nodes,
1273 [DPO_PROTO_IP6] = lookup_src_ip6_nodes,
1274 [DPO_PROTO_MPLS] = NULL,
1277 const static char* const lookup_dst_ip4_nodes[] =
1282 const static char* const lookup_dst_ip6_nodes[] =
1287 const static char* const lookup_dst_mpls_nodes[] =
1292 const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
1294 [DPO_PROTO_IP4] = lookup_dst_ip4_nodes,
1295 [DPO_PROTO_IP6] = lookup_dst_ip6_nodes,
1296 [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
1299 const static char* const lookup_dst_mcast_ip4_nodes[] =
1301 "lookup-ip4-dst-mcast",
1304 const static char* const lookup_dst_mcast_ip6_nodes[] =
1306 "lookup-ip6-dst-mcast",
1309 const static char* const * const lookup_dst_mcast_nodes[DPO_PROTO_NUM] =
1311 [DPO_PROTO_IP4] = lookup_dst_mcast_ip4_nodes,
1312 [DPO_PROTO_IP6] = lookup_dst_mcast_ip6_nodes,
1315 const static char* const lookup_dst_from_interface_ip4_nodes[] =
1317 "lookup-ip4-dst-itf",
1320 const static char* const lookup_dst_from_interface_ip6_nodes[] =
1322 "lookup-ip6-dst-itf",
1325 const static char* const lookup_dst_from_interface_mpls_nodes[] =
1327 "lookup-mpls-dst-itf",
1330 const static char* const * const lookup_dst_from_interface_nodes[DPO_PROTO_NUM] =
1332 [DPO_PROTO_IP4] = lookup_dst_from_interface_ip4_nodes,
1333 [DPO_PROTO_IP6] = lookup_dst_from_interface_ip6_nodes,
1334 [DPO_PROTO_MPLS] = lookup_dst_from_interface_mpls_nodes,
1339 lookup_dpo_module_init (void)
1341 dpo_register(DPO_LOOKUP, &lkd_vft_w_mem_show, NULL);
1344 * There are various sorts of lookup; src or dst addr v4 /v6 etc.
1345 * there isn't an object type for each (there is only the lookup_dpo_t),
1346 * but, for performance reasons, there is a data plane function, and hence
1347 * VLIB node for each. VLIB graph node construction is based on DPO types
1348 * so we create sub-types.
1350 lookup_dpo_sub_types[LOOKUP_SUB_TYPE_SRC] =
1351 dpo_register_new_type(&lkd_vft, lookup_src_nodes);
1352 lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
1353 dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
1354 lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST] =
1355 dpo_register_new_type(&lkd_vft, lookup_dst_mcast_nodes);
1356 lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
1357 dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes);