2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/adj/adj_nbr.h>
17 #include <vnet/adj/adj_internal.h>
18 #include <vnet/adj/adj_l2.h>
19 #include <vnet/adj/adj_nsh.h>
20 #include <vnet/adj/adj_midchain.h>
21 #include <vnet/ethernet/arp_packet.h>
22 #include <vnet/dpo/drop_dpo.h>
23 #include <vnet/dpo/load_balance.h>
24 #include <vnet/fib/fib_walk.h>
25 #include <vnet/fib/fib_entry.h>
28 * The two midchain tx feature node indices
30 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
31 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
34 * @brief Trace data for packets traversing the midchain tx node
36 typedef struct adj_midchain_tx_trace_t_
39 * @brief the midchain adj we are traversing
42 } adj_midchain_tx_trace_t;
45 adj_midchain_tx_inline (vlib_main_t * vm,
46 vlib_node_runtime_t * node,
50 u32 * from, * to_next, n_left_from, n_left_to_next;
52 vnet_main_t *vnm = vnet_get_main ();
53 vnet_interface_main_t *im = &vnm->interface_main;
54 u32 thread_index = vm->thread_index;
56 /* Vector of buffer / pkt indices we're supposed to process */
57 from = vlib_frame_vector_args (frame);
59 /* Number of buffers / pkts */
60 n_left_from = frame->n_vectors;
62 /* Speculatively send the first buffer to the last disposition we used */
63 next_index = node->cached_next_index;
65 while (n_left_from > 0)
67 /* set up to enqueue to our disposition with index = next_index */
68 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
70 while (n_left_from >= 8 && n_left_to_next > 4)
72 u32 bi0, adj_index0, next0;
73 const ip_adjacency_t * adj0;
76 u32 bi1, adj_index1, next1;
77 const ip_adjacency_t * adj1;
80 u32 bi2, adj_index2, next2;
81 const ip_adjacency_t * adj2;
84 u32 bi3, adj_index3, next3;
85 const ip_adjacency_t * adj3;
89 /* Prefetch next iteration. */
91 vlib_buffer_t * p4, * p5;
92 vlib_buffer_t * p6, * p7;
94 p4 = vlib_get_buffer (vm, from[4]);
95 p5 = vlib_get_buffer (vm, from[5]);
96 p6 = vlib_get_buffer (vm, from[6]);
97 p7 = vlib_get_buffer (vm, from[7]);
99 vlib_prefetch_buffer_header (p4, LOAD);
100 vlib_prefetch_buffer_header (p5, LOAD);
101 vlib_prefetch_buffer_header (p6, LOAD);
102 vlib_prefetch_buffer_header (p7, LOAD);
119 b0 = vlib_get_buffer(vm, bi0);
120 b1 = vlib_get_buffer(vm, bi1);
121 b2 = vlib_get_buffer(vm, bi2);
122 b3 = vlib_get_buffer(vm, bi3);
124 /* Follow the DPO on which the midchain is stacked */
125 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
126 adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
127 adj_index2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
128 adj_index3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
130 adj0 = adj_get(adj_index0);
131 adj1 = adj_get(adj_index1);
132 adj2 = adj_get(adj_index2);
133 adj3 = adj_get(adj_index3);
135 dpo0 = &adj0->sub_type.midchain.next_dpo;
136 dpo1 = &adj1->sub_type.midchain.next_dpo;
137 dpo2 = &adj2->sub_type.midchain.next_dpo;
138 dpo3 = &adj3->sub_type.midchain.next_dpo;
140 next0 = dpo0->dpoi_next_node;
141 next1 = dpo1->dpoi_next_node;
142 next2 = dpo2->dpoi_next_node;
143 next3 = dpo3->dpoi_next_node;
145 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
146 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
147 vnet_buffer(b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
148 vnet_buffer(b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
152 vlib_increment_combined_counter (im->combined_sw_if_counters
153 + VNET_INTERFACE_COUNTER_TX,
155 adj0->rewrite_header.sw_if_index,
157 vlib_buffer_length_in_chain (vm, b0));
158 vlib_increment_combined_counter (im->combined_sw_if_counters
159 + VNET_INTERFACE_COUNTER_TX,
161 adj1->rewrite_header.sw_if_index,
163 vlib_buffer_length_in_chain (vm, b1));
164 vlib_increment_combined_counter (im->combined_sw_if_counters
165 + VNET_INTERFACE_COUNTER_TX,
167 adj2->rewrite_header.sw_if_index,
169 vlib_buffer_length_in_chain (vm, b2));
170 vlib_increment_combined_counter (im->combined_sw_if_counters
171 + VNET_INTERFACE_COUNTER_TX,
173 adj3->rewrite_header.sw_if_index,
175 vlib_buffer_length_in_chain (vm, b3));
178 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
180 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
184 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
186 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
190 if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
192 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
196 if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
198 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
203 vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
204 to_next, n_left_to_next,
206 next0, next1, next2, next3);
208 while (n_left_from > 0 && n_left_to_next > 0)
210 u32 bi0, adj_index0, next0;
211 const ip_adjacency_t * adj0;
212 const dpo_id_t *dpo0;
222 b0 = vlib_get_buffer(vm, bi0);
224 /* Follow the DPO on which the midchain is stacked */
225 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
226 adj0 = adj_get(adj_index0);
227 dpo0 = &adj0->sub_type.midchain.next_dpo;
228 next0 = dpo0->dpoi_next_node;
229 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
233 vlib_increment_combined_counter (im->combined_sw_if_counters
234 + VNET_INTERFACE_COUNTER_TX,
236 adj0->rewrite_header.sw_if_index,
238 vlib_buffer_length_in_chain (vm, b0));
241 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
243 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
248 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
249 to_next, n_left_to_next,
253 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
256 return frame->n_vectors;
260 format_adj_midchain_tx_trace (u8 * s, va_list * args)
262 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
263 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
264 adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
266 s = format(s, "adj-midchain:[%d]:%U", tr->ai,
267 format_ip_adjacency, tr->ai,
268 FORMAT_IP_ADJACENCY_NONE);
274 adj_midchain_tx (vlib_main_t * vm,
275 vlib_node_runtime_t * node,
276 vlib_frame_t * frame)
278 return (adj_midchain_tx_inline(vm, node, frame, 1));
281 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
282 .function = adj_midchain_tx,
283 .name = "adj-midchain-tx",
284 .vector_size = sizeof (u32),
286 .format_trace = format_adj_midchain_tx_trace,
295 adj_midchain_tx_no_count (vlib_main_t * vm,
296 vlib_node_runtime_t * node,
297 vlib_frame_t * frame)
299 return (adj_midchain_tx_inline(vm, node, frame, 0));
302 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
303 .function = adj_midchain_tx_no_count,
304 .name = "adj-midchain-tx-no-count",
305 .vector_size = sizeof (u32),
307 .format_trace = format_adj_midchain_tx_trace,
315 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
316 .arc_name = "ip4-output",
317 .node_name = "adj-midchain-tx",
318 .runs_before = VNET_FEATURES ("interface-output"),
319 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
321 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
322 .arc_name = "ip4-output",
323 .node_name = "adj-midchain-tx-no-count",
324 .runs_before = VNET_FEATURES ("interface-output"),
325 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
327 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
328 .arc_name = "ip6-output",
329 .node_name = "adj-midchain-tx",
330 .runs_before = VNET_FEATURES ("interface-output"),
331 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
333 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
334 .arc_name = "ip6-output",
335 .node_name = "adj-midchain-tx-no-count",
336 .runs_before = VNET_FEATURES ("interface-output"),
337 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
339 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
340 .arc_name = "mpls-output",
341 .node_name = "adj-midchain-tx",
342 .runs_before = VNET_FEATURES ("interface-output"),
343 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
345 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
346 .arc_name = "mpls-output",
347 .node_name = "adj-midchain-tx-no-count",
348 .runs_before = VNET_FEATURES ("interface-output"),
349 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
351 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
352 .arc_name = "ethernet-output",
353 .node_name = "adj-midchain-tx",
354 .runs_before = VNET_FEATURES ("error-drop"),
355 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
357 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
358 .arc_name = "ethernet-output",
359 .node_name = "adj-midchain-tx-no-count",
360 .runs_before = VNET_FEATURES ("error-drop"),
361 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
363 VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = {
364 .arc_name = "nsh-output",
365 .node_name = "adj-midchain-tx",
366 .runs_before = VNET_FEATURES ("error-drop"),
367 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH],
369 VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = {
370 .arc_name = "nsh-output",
371 .node_name = "adj-midchain-tx-no-count",
372 .runs_before = VNET_FEATURES ("error-drop"),
373 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH],
377 adj_get_midchain_node (vnet_link_t link)
381 return (ip4_midchain_node.index);
383 return (ip6_midchain_node.index);
385 return (mpls_midchain_node.index);
386 case VNET_LINK_ETHERNET:
387 return (adj_l2_midchain_node.index);
389 return (adj_nsh_midchain_node.index);
398 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
401 switch (adj->ia_link)
405 arc = ip4_main.lookup_main.output_feature_arc_index;
410 arc = ip6_main.lookup_main.output_feature_arc_index;
415 arc = mpls_main.output_feature_arc_index;
418 case VNET_LINK_ETHERNET:
420 arc = ethernet_main.output_feature_arc_index;
425 arc = nsh_main_dummy.output_feature_arc_index;
433 ASSERT (arc != (u8) ~0);
439 adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
441 return ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
442 adj_midchain_tx_no_count_node.index :
443 adj_midchain_tx_node.index);
447 adj_nbr_midchain_get_feature_node (ip_adjacency_t *adj)
449 if (adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT)
451 return (adj_midchain_tx_no_count_feature_node[adj->ia_link]);
454 return (adj_midchain_tx_feature_node[adj->ia_link]);
460 * Setup the adj as a mid-chain
463 adj_midchain_setup (adj_index_t adj_index,
464 adj_midchain_fixup_t fixup,
468 u32 feature_index, tx_node;
472 ASSERT(ADJ_INDEX_INVALID != adj_index);
474 adj = adj_get(adj_index);
476 adj->sub_type.midchain.fixup_func = fixup;
477 adj->sub_type.midchain.fixup_data = data;
478 adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID;
479 adj->ia_flags |= flags;
481 arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
482 feature_index = adj_nbr_midchain_get_feature_node(adj);
483 tx_node = adj_nbr_midchain_get_tx_node(adj);
485 vnet_feature_enable_disable_with_index (arc_index, feature_index,
486 adj->rewrite_header.sw_if_index,
487 1 /* enable */, 0, 0);
490 * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
491 * The graph arc used/created here is from the midchain-tx node to the
492 * child's registered node. This is because post adj processing the next
493 * node are any output features, then the midchain-tx. from there we
494 * need to get to the stacked child's node.
496 dpo_stack_from_node(tx_node,
497 &adj->sub_type.midchain.next_dpo,
498 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
502 * adj_nbr_midchain_update_rewrite
504 * Update the adjacency's rewrite string. A NULL string implies the
505 * rewrite is reset (i.e. when ARP/ND etnry is gone).
506 * NB: the adj being updated may be handling traffic in the DP.
509 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
510 adj_midchain_fixup_t fixup,
511 const void *fixup_data,
517 ASSERT(ADJ_INDEX_INVALID != adj_index);
519 adj = adj_get(adj_index);
522 * one time only update. since we don't support chainging the tunnel
523 * src,dst, this is all we need.
525 ASSERT((adj->lookup_next_index == IP_LOOKUP_NEXT_ARP) ||
526 (adj->lookup_next_index == IP_LOOKUP_NEXT_GLEAN) ||
527 (adj->lookup_next_index == IP_LOOKUP_NEXT_BCAST));
530 * tunnels can always provide a rewrite.
532 ASSERT(NULL != rewrite);
534 adj_midchain_setup(adj_index, fixup, fixup_data, flags);
537 * update the rewirte with the workers paused.
539 adj_nbr_update_rewrite_internal(adj,
540 IP_LOOKUP_NEXT_MIDCHAIN,
541 adj_get_midchain_node(adj->ia_link),
542 adj_nbr_midchain_get_tx_node(adj),
547 * adj_nbr_midchain_unstack
549 * Unstack the adj. stack it on drop
552 adj_nbr_midchain_unstack (adj_index_t adj_index)
554 fib_node_index_t *entry_indicies, tmp;
557 ASSERT(ADJ_INDEX_INVALID != adj_index);
558 adj = adj_get (adj_index);
561 * check to see if this unstacking breaks a recursion loop
563 entry_indicies = NULL;
564 tmp = adj->sub_type.midchain.fei;
565 adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID;
567 if (FIB_NODE_INDEX_INVALID != tmp)
569 fib_entry_recursive_loop_detect(tmp, &entry_indicies);
570 vec_free(entry_indicies);
576 dpo_stack(DPO_ADJACENCY_MIDCHAIN,
577 vnet_link_to_dpo_proto(adj->ia_link),
578 &adj->sub_type.midchain.next_dpo,
579 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
580 CLIB_MEMORY_BARRIER();
584 adj_nbr_midchain_stack_on_fib_entry (adj_index_t ai,
585 fib_node_index_t fei,
586 fib_forward_chain_type_t fct)
588 fib_node_index_t *entry_indicies;
589 dpo_id_t tmp = DPO_INVALID;
595 * check to see if this stacking will form a recursion loop
597 entry_indicies = NULL;
598 adj->sub_type.midchain.fei = fei;
600 if (fib_entry_recursive_loop_detect(adj->sub_type.midchain.fei, &entry_indicies))
603 * loop formed, stack on the drop.
605 dpo_copy(&tmp, drop_dpo_get(fib_forw_chain_type_to_dpo_proto(fct)));
609 fib_entry_contribute_forwarding (fei, fct, &tmp);
611 if ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_IP_STACK) &&
612 (DPO_LOAD_BALANCE == tmp.dpoi_type))
615 * do that hash now and stack on the choice.
616 * If the choice is an incomplete adj then we will need a poke when
617 * it becomes complete. This happens since the adj update walk propagates
618 * as far a recursive paths.
620 const dpo_id_t *choice;
624 lb = load_balance_get (tmp.dpoi_index);
626 if (FIB_FORW_CHAIN_TYPE_UNICAST_IP4 == fct)
628 hash = ip4_compute_flow_hash ((ip4_header_t *) adj_get_rewrite (ai),
631 else if (FIB_FORW_CHAIN_TYPE_UNICAST_IP6 == fct)
633 hash = ip6_compute_flow_hash ((ip6_header_t *) adj_get_rewrite (ai),
642 choice = load_balance_get_bucket_i (lb, hash & lb->lb_n_buckets_minus_1);
643 dpo_copy (&tmp, choice);
646 adj_nbr_midchain_stack (ai, &tmp);
648 vec_free(entry_indicies);
652 * adj_nbr_midchain_stack
655 adj_nbr_midchain_stack (adj_index_t adj_index,
656 const dpo_id_t *next)
660 ASSERT(ADJ_INDEX_INVALID != adj_index);
662 adj = adj_get(adj_index);
664 ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) ||
665 (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index));
667 dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
668 &adj->sub_type.midchain.next_dpo,
673 adj_ndr_midchain_recursive_loop_detect (adj_index_t ai,
674 fib_node_index_t **entry_indicies)
676 fib_node_index_t *entry_index, *entries;
677 ip_adjacency_t * adj;
680 entries = *entry_indicies;
682 vec_foreach(entry_index, entries)
684 if (*entry_index == adj->sub_type.midchain.fei)
687 * The entry this midchain links to is already in the set
688 * of visisted entries, this is a loop
690 adj->ia_flags |= ADJ_FLAG_MIDCHAIN_LOOPED;
695 adj->ia_flags &= ~ADJ_FLAG_MIDCHAIN_LOOPED;
700 format_adj_midchain (u8* s, va_list *ap)
702 index_t index = va_arg(*ap, index_t);
703 u32 indent = va_arg(*ap, u32);
704 ip_adjacency_t * adj = adj_get(index);
706 s = format (s, "%U", format_vnet_link, adj->ia_link);
707 s = format (s, " via %U",
708 format_ip46_address, &adj->sub_type.nbr.next_hop,
709 adj_proto_to_46(adj->ia_nh_proto));
710 s = format (s, " %U",
712 &adj->rewrite_header, sizeof (adj->rewrite_data), indent);
713 s = format (s, "\n%Ustacked-on",
714 format_white_space, indent);
716 if (FIB_NODE_INDEX_INVALID != adj->sub_type.midchain.fei)
718 s = format (s, " entry:%d", adj->sub_type.midchain.fei);
721 s = format (s, ":\n%U%U",
722 format_white_space, indent+2,
723 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
729 adj_dpo_lock (dpo_id_t *dpo)
731 adj_lock(dpo->dpoi_index);
734 adj_dpo_unlock (dpo_id_t *dpo)
736 adj_unlock(dpo->dpoi_index);
739 const static dpo_vft_t adj_midchain_dpo_vft = {
740 .dv_lock = adj_dpo_lock,
741 .dv_unlock = adj_dpo_unlock,
742 .dv_format = format_adj_midchain,
743 .dv_get_urpf = adj_dpo_get_urpf,
747 * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
750 * this means that these graph nodes are ones from which a midchain is the
751 * parent object in the DPO-graph.
753 const static char* const midchain_ip4_nodes[] =
758 const static char* const midchain_ip6_nodes[] =
763 const static char* const midchain_mpls_nodes[] =
768 const static char* const midchain_ethernet_nodes[] =
773 const static char* const midchain_nsh_nodes[] =
779 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
781 [DPO_PROTO_IP4] = midchain_ip4_nodes,
782 [DPO_PROTO_IP6] = midchain_ip6_nodes,
783 [DPO_PROTO_MPLS] = midchain_mpls_nodes,
784 [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
785 [DPO_PROTO_NSH] = midchain_nsh_nodes,
789 adj_midchain_module_init (void)
791 dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);