2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/adj/adj_nbr.h>
17 #include <vnet/adj/adj_internal.h>
18 #include <vnet/adj/adj_l2.h>
19 #include <vnet/adj/adj_nsh.h>
20 #include <vnet/adj/adj_midchain.h>
21 #include <vnet/ethernet/arp_packet.h>
22 #include <vnet/dpo/drop_dpo.h>
23 #include <vnet/dpo/load_balance.h>
24 #include <vnet/fib/fib_walk.h>
25 #include <vnet/fib/fib_entry.h>
28 * The two midchain tx feature node indices
30 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
31 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
34 * @brief Trace data for packets traversing the midchain tx node
36 typedef struct adj_midchain_tx_trace_t_
39 * @brief the midchain adj we are traversing
42 } adj_midchain_tx_trace_t;
45 adj_midchain_tx_inline (vlib_main_t * vm,
46 vlib_node_runtime_t * node,
50 u32 * from, * to_next, n_left_from, n_left_to_next;
52 vnet_main_t *vnm = vnet_get_main ();
53 vnet_interface_main_t *im = &vnm->interface_main;
54 u32 thread_index = vm->thread_index;
56 /* Vector of buffer / pkt indices we're supposed to process */
57 from = vlib_frame_vector_args (frame);
59 /* Number of buffers / pkts */
60 n_left_from = frame->n_vectors;
62 /* Speculatively send the first buffer to the last disposition we used */
63 next_index = node->cached_next_index;
65 while (n_left_from > 0)
67 /* set up to enqueue to our disposition with index = next_index */
68 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
70 while (n_left_from >= 8 && n_left_to_next > 4)
72 const ip_adjacency_t *adj0, *adj1, *adj2, *adj3;
73 const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
74 vlib_buffer_t * b0, *b1, *b2, *b3;
75 u32 bi0, adj_index0, next0;
76 u32 bi1, adj_index1, next1;
77 u32 bi2, adj_index2, next2;
78 u32 bi3, adj_index3, next3;
80 /* Prefetch next iteration. */
82 vlib_buffer_t * p4, * p5;
83 vlib_buffer_t * p6, * p7;
85 p4 = vlib_get_buffer (vm, from[4]);
86 p5 = vlib_get_buffer (vm, from[5]);
87 p6 = vlib_get_buffer (vm, from[6]);
88 p7 = vlib_get_buffer (vm, from[7]);
90 vlib_prefetch_buffer_header (p4, LOAD);
91 vlib_prefetch_buffer_header (p5, LOAD);
92 vlib_prefetch_buffer_header (p6, LOAD);
93 vlib_prefetch_buffer_header (p7, LOAD);
110 b0 = vlib_get_buffer(vm, bi0);
111 b1 = vlib_get_buffer(vm, bi1);
112 b2 = vlib_get_buffer(vm, bi2);
113 b3 = vlib_get_buffer(vm, bi3);
115 /* Follow the DPO on which the midchain is stacked */
116 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
117 adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
118 adj_index2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
119 adj_index3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
121 adj0 = adj_get(adj_index0);
122 adj1 = adj_get(adj_index1);
123 adj2 = adj_get(adj_index2);
124 adj3 = adj_get(adj_index3);
126 dpo0 = &adj0->sub_type.midchain.next_dpo;
127 dpo1 = &adj1->sub_type.midchain.next_dpo;
128 dpo2 = &adj2->sub_type.midchain.next_dpo;
129 dpo3 = &adj3->sub_type.midchain.next_dpo;
131 next0 = dpo0->dpoi_next_node;
132 next1 = dpo1->dpoi_next_node;
133 next2 = dpo2->dpoi_next_node;
134 next3 = dpo3->dpoi_next_node;
136 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
137 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
138 vnet_buffer(b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
139 vnet_buffer(b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
143 vlib_increment_combined_counter (im->combined_sw_if_counters
144 + VNET_INTERFACE_COUNTER_TX,
146 adj0->rewrite_header.sw_if_index,
148 vlib_buffer_length_in_chain (vm, b0));
149 vlib_increment_combined_counter (im->combined_sw_if_counters
150 + VNET_INTERFACE_COUNTER_TX,
152 adj1->rewrite_header.sw_if_index,
154 vlib_buffer_length_in_chain (vm, b1));
155 vlib_increment_combined_counter (im->combined_sw_if_counters
156 + VNET_INTERFACE_COUNTER_TX,
158 adj2->rewrite_header.sw_if_index,
160 vlib_buffer_length_in_chain (vm, b2));
161 vlib_increment_combined_counter (im->combined_sw_if_counters
162 + VNET_INTERFACE_COUNTER_TX,
164 adj3->rewrite_header.sw_if_index,
166 vlib_buffer_length_in_chain (vm, b3));
169 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
171 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
175 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
177 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
181 if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
183 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
187 if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
189 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
194 vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
195 to_next, n_left_to_next,
197 next0, next1, next2, next3);
199 while (n_left_from > 0 && n_left_to_next > 0)
201 u32 bi0, adj_index0, next0;
202 const ip_adjacency_t * adj0;
203 const dpo_id_t *dpo0;
213 b0 = vlib_get_buffer(vm, bi0);
215 /* Follow the DPO on which the midchain is stacked */
216 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
217 adj0 = adj_get(adj_index0);
218 dpo0 = &adj0->sub_type.midchain.next_dpo;
219 next0 = dpo0->dpoi_next_node;
220 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
224 vlib_increment_combined_counter (im->combined_sw_if_counters
225 + VNET_INTERFACE_COUNTER_TX,
227 adj0->rewrite_header.sw_if_index,
229 vlib_buffer_length_in_chain (vm, b0));
232 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
234 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
239 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
240 to_next, n_left_to_next,
244 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
247 return frame->n_vectors;
251 format_adj_midchain_tx_trace (u8 * s, va_list * args)
253 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
254 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
255 adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
257 s = format(s, "adj-midchain:[%d]:%U", tr->ai,
258 format_ip_adjacency, tr->ai,
259 FORMAT_IP_ADJACENCY_NONE);
265 adj_midchain_tx (vlib_main_t * vm,
266 vlib_node_runtime_t * node,
267 vlib_frame_t * frame)
269 return (adj_midchain_tx_inline(vm, node, frame, 1));
272 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
273 .function = adj_midchain_tx,
274 .name = "adj-midchain-tx",
275 .vector_size = sizeof (u32),
277 .format_trace = format_adj_midchain_tx_trace,
286 adj_midchain_tx_no_count (vlib_main_t * vm,
287 vlib_node_runtime_t * node,
288 vlib_frame_t * frame)
290 return (adj_midchain_tx_inline(vm, node, frame, 0));
293 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
294 .function = adj_midchain_tx_no_count,
295 .name = "adj-midchain-tx-no-count",
296 .vector_size = sizeof (u32),
298 .format_trace = format_adj_midchain_tx_trace,
306 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
307 .arc_name = "ip4-output",
308 .node_name = "adj-midchain-tx",
309 .runs_before = VNET_FEATURES ("interface-output"),
310 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
312 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
313 .arc_name = "ip4-output",
314 .node_name = "adj-midchain-tx-no-count",
315 .runs_before = VNET_FEATURES ("interface-output"),
316 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
318 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
319 .arc_name = "ip6-output",
320 .node_name = "adj-midchain-tx",
321 .runs_before = VNET_FEATURES ("interface-output"),
322 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
324 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
325 .arc_name = "ip6-output",
326 .node_name = "adj-midchain-tx-no-count",
327 .runs_before = VNET_FEATURES ("interface-output"),
328 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
330 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
331 .arc_name = "mpls-output",
332 .node_name = "adj-midchain-tx",
333 .runs_before = VNET_FEATURES ("interface-output"),
334 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
336 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
337 .arc_name = "mpls-output",
338 .node_name = "adj-midchain-tx-no-count",
339 .runs_before = VNET_FEATURES ("interface-output"),
340 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
342 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
343 .arc_name = "ethernet-output",
344 .node_name = "adj-midchain-tx",
345 .runs_before = VNET_FEATURES ("error-drop"),
346 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
348 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
349 .arc_name = "ethernet-output",
350 .node_name = "adj-midchain-tx-no-count",
351 .runs_before = VNET_FEATURES ("error-drop"),
352 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
354 VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = {
355 .arc_name = "nsh-output",
356 .node_name = "adj-midchain-tx",
357 .runs_before = VNET_FEATURES ("error-drop"),
358 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH],
360 VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = {
361 .arc_name = "nsh-output",
362 .node_name = "adj-midchain-tx-no-count",
363 .runs_before = VNET_FEATURES ("error-drop"),
364 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH],
368 adj_get_midchain_node (vnet_link_t link)
372 return (ip4_midchain_node.index);
374 return (ip6_midchain_node.index);
376 return (mpls_midchain_node.index);
377 case VNET_LINK_ETHERNET:
378 return (adj_l2_midchain_node.index);
380 return (adj_nsh_midchain_node.index);
389 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
392 switch (adj->ia_link)
396 arc = ip4_main.lookup_main.output_feature_arc_index;
401 arc = ip6_main.lookup_main.output_feature_arc_index;
406 arc = mpls_main.output_feature_arc_index;
409 case VNET_LINK_ETHERNET:
411 arc = ethernet_main.output_feature_arc_index;
416 arc = nsh_main_dummy.output_feature_arc_index;
424 ASSERT (arc != (u8) ~0);
430 adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
432 return ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
433 adj_midchain_tx_no_count_node.index :
434 adj_midchain_tx_node.index);
438 adj_nbr_midchain_get_feature_node (ip_adjacency_t *adj)
440 if (adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT)
442 return (adj_midchain_tx_no_count_feature_node[adj->ia_link]);
445 return (adj_midchain_tx_feature_node[adj->ia_link]);
451 * Setup the adj as a mid-chain
454 adj_midchain_setup (adj_index_t adj_index,
455 adj_midchain_fixup_t fixup,
459 u32 feature_index, tx_node;
463 ASSERT(ADJ_INDEX_INVALID != adj_index);
465 adj = adj_get(adj_index);
467 adj->sub_type.midchain.fixup_func = fixup;
468 adj->sub_type.midchain.fixup_data = data;
469 adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID;
470 adj->ia_flags |= flags;
472 arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
473 feature_index = adj_nbr_midchain_get_feature_node(adj);
474 tx_node = adj_nbr_midchain_get_tx_node(adj);
476 vnet_feature_enable_disable_with_index (arc_index, feature_index,
477 adj->rewrite_header.sw_if_index,
478 1 /* enable */, 0, 0);
481 * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
482 * The graph arc used/created here is from the midchain-tx node to the
483 * child's registered node. This is because post adj processing the next
484 * node are any output features, then the midchain-tx. from there we
485 * need to get to the stacked child's node.
487 dpo_stack_from_node(tx_node,
488 &adj->sub_type.midchain.next_dpo,
489 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
493 * adj_nbr_midchain_update_rewrite
495 * Update the adjacency's rewrite string. A NULL string implies the
496 * rewrite is reset (i.e. when ARP/ND entry is gone).
497 * NB: the adj being updated may be handling traffic in the DP.
500 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
501 adj_midchain_fixup_t fixup,
502 const void *fixup_data,
508 ASSERT(ADJ_INDEX_INVALID != adj_index);
510 adj = adj_get(adj_index);
513 * one time only update. since we don't support changing the tunnel
514 * src,dst, this is all we need.
516 ASSERT((adj->lookup_next_index == IP_LOOKUP_NEXT_ARP) ||
517 (adj->lookup_next_index == IP_LOOKUP_NEXT_GLEAN) ||
518 (adj->lookup_next_index == IP_LOOKUP_NEXT_BCAST));
520 adj_midchain_setup(adj_index, fixup, fixup_data, flags);
523 * update the rewrite with the workers paused.
525 adj_nbr_update_rewrite_internal(adj,
526 IP_LOOKUP_NEXT_MIDCHAIN,
527 adj_get_midchain_node(adj->ia_link),
528 adj_nbr_midchain_get_tx_node(adj),
533 * adj_nbr_midchain_unstack
535 * Unstack the adj. stack it on drop
538 adj_nbr_midchain_unstack (adj_index_t adj_index)
540 fib_node_index_t *entry_indicies, tmp;
543 ASSERT(ADJ_INDEX_INVALID != adj_index);
544 adj = adj_get (adj_index);
547 * check to see if this unstacking breaks a recursion loop
549 entry_indicies = NULL;
550 tmp = adj->sub_type.midchain.fei;
551 adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID;
553 if (FIB_NODE_INDEX_INVALID != tmp)
555 fib_entry_recursive_loop_detect(tmp, &entry_indicies);
556 vec_free(entry_indicies);
562 dpo_stack(DPO_ADJACENCY_MIDCHAIN,
563 vnet_link_to_dpo_proto(adj->ia_link),
564 &adj->sub_type.midchain.next_dpo,
565 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
566 CLIB_MEMORY_BARRIER();
570 adj_nbr_midchain_stack_on_fib_entry (adj_index_t ai,
571 fib_node_index_t fei,
572 fib_forward_chain_type_t fct)
574 fib_node_index_t *entry_indicies;
575 dpo_id_t tmp = DPO_INVALID;
581 * check to see if this stacking will form a recursion loop
583 entry_indicies = NULL;
584 adj->sub_type.midchain.fei = fei;
586 if (fib_entry_recursive_loop_detect(adj->sub_type.midchain.fei, &entry_indicies))
589 * loop formed, stack on the drop.
591 dpo_copy(&tmp, drop_dpo_get(fib_forw_chain_type_to_dpo_proto(fct)));
595 fib_entry_contribute_forwarding (fei, fct, &tmp);
597 if ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_IP_STACK) &&
598 (DPO_LOAD_BALANCE == tmp.dpoi_type))
601 * do that hash now and stack on the choice.
602 * If the choice is an incomplete adj then we will need a poke when
603 * it becomes complete. This happens since the adj update walk propagates
604 * as far a recursive paths.
606 const dpo_id_t *choice;
610 lb = load_balance_get (tmp.dpoi_index);
612 if (FIB_FORW_CHAIN_TYPE_UNICAST_IP4 == fct)
614 hash = ip4_compute_flow_hash ((ip4_header_t *) adj_get_rewrite (ai),
617 else if (FIB_FORW_CHAIN_TYPE_UNICAST_IP6 == fct)
619 hash = ip6_compute_flow_hash ((ip6_header_t *) adj_get_rewrite (ai),
628 choice = load_balance_get_bucket_i (lb, hash & lb->lb_n_buckets_minus_1);
629 dpo_copy (&tmp, choice);
632 adj_nbr_midchain_stack (ai, &tmp);
634 vec_free(entry_indicies);
638 * adj_nbr_midchain_stack
641 adj_nbr_midchain_stack (adj_index_t adj_index,
642 const dpo_id_t *next)
646 ASSERT(ADJ_INDEX_INVALID != adj_index);
648 adj = adj_get(adj_index);
650 ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) ||
651 (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index));
653 dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
654 &adj->sub_type.midchain.next_dpo,
659 adj_ndr_midchain_recursive_loop_detect (adj_index_t ai,
660 fib_node_index_t **entry_indicies)
662 fib_node_index_t *entry_index, *entries;
663 ip_adjacency_t * adj;
666 entries = *entry_indicies;
668 vec_foreach(entry_index, entries)
670 if (*entry_index == adj->sub_type.midchain.fei)
673 * The entry this midchain links to is already in the set
674 * of visited entries, this is a loop
676 adj->ia_flags |= ADJ_FLAG_MIDCHAIN_LOOPED;
681 adj->ia_flags &= ~ADJ_FLAG_MIDCHAIN_LOOPED;
686 format_adj_midchain (u8* s, va_list *ap)
688 index_t index = va_arg(*ap, index_t);
689 u32 indent = va_arg(*ap, u32);
690 ip_adjacency_t * adj = adj_get(index);
692 s = format (s, "%U", format_vnet_link, adj->ia_link);
693 if (adj->rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)
694 s = format(s, " [features]");
695 s = format (s, " via %U",
696 format_ip46_address, &adj->sub_type.nbr.next_hop,
697 adj_proto_to_46(adj->ia_nh_proto));
698 s = format (s, " %U",
700 &adj->rewrite_header, sizeof (adj->rewrite_data), indent);
701 s = format (s, "\n%Ustacked-on",
702 format_white_space, indent);
704 if (FIB_NODE_INDEX_INVALID != adj->sub_type.midchain.fei)
706 s = format (s, " entry:%d", adj->sub_type.midchain.fei);
709 s = format (s, ":\n%U%U",
710 format_white_space, indent+2,
711 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
717 adj_dpo_lock (dpo_id_t *dpo)
719 adj_lock(dpo->dpoi_index);
722 adj_dpo_unlock (dpo_id_t *dpo)
724 adj_unlock(dpo->dpoi_index);
727 const static dpo_vft_t adj_midchain_dpo_vft = {
728 .dv_lock = adj_dpo_lock,
729 .dv_unlock = adj_dpo_unlock,
730 .dv_format = format_adj_midchain,
731 .dv_get_urpf = adj_dpo_get_urpf,
735 * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
738 * this means that these graph nodes are ones from which a midchain is the
739 * parent object in the DPO-graph.
741 const static char* const midchain_ip4_nodes[] =
746 const static char* const midchain_ip6_nodes[] =
751 const static char* const midchain_mpls_nodes[] =
756 const static char* const midchain_ethernet_nodes[] =
761 const static char* const midchain_nsh_nodes[] =
767 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
769 [DPO_PROTO_IP4] = midchain_ip4_nodes,
770 [DPO_PROTO_IP6] = midchain_ip6_nodes,
771 [DPO_PROTO_MPLS] = midchain_mpls_nodes,
772 [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
773 [DPO_PROTO_NSH] = midchain_nsh_nodes,
777 adj_midchain_module_init (void)
779 dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);