2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/adj/adj_nbr.h>
17 #include <vnet/adj/adj_internal.h>
18 #include <vnet/adj/adj_l2.h>
19 #include <vnet/adj/adj_nsh.h>
20 #include <vnet/adj/adj_midchain.h>
21 #include <vnet/ethernet/arp_packet.h>
22 #include <vnet/dpo/drop_dpo.h>
23 #include <vnet/fib/fib_walk.h>
26 * The two midchain tx feature node indices
28 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
29 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
32 * @brief Trace data for packets traversing the midchain tx node
34 typedef struct adj_midchain_tx_trace_t_
37 * @brief the midchain adj we are traversing
40 } adj_midchain_tx_trace_t;
43 adj_midchain_tx_inline (vlib_main_t * vm,
44 vlib_node_runtime_t * node,
48 u32 * from, * to_next, n_left_from, n_left_to_next;
50 vnet_main_t *vnm = vnet_get_main ();
51 vnet_interface_main_t *im = &vnm->interface_main;
52 u32 thread_index = vm->thread_index;
54 /* Vector of buffer / pkt indices we're supposed to process */
55 from = vlib_frame_vector_args (frame);
57 /* Number of buffers / pkts */
58 n_left_from = frame->n_vectors;
60 /* Speculatively send the first buffer to the last disposition we used */
61 next_index = node->cached_next_index;
63 while (n_left_from > 0)
65 /* set up to enqueue to our disposition with index = next_index */
66 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
68 while (n_left_from >= 8 && n_left_to_next > 4)
70 u32 bi0, adj_index0, next0;
71 const ip_adjacency_t * adj0;
74 u32 bi1, adj_index1, next1;
75 const ip_adjacency_t * adj1;
78 u32 bi2, adj_index2, next2;
79 const ip_adjacency_t * adj2;
82 u32 bi3, adj_index3, next3;
83 const ip_adjacency_t * adj3;
87 /* Prefetch next iteration. */
89 vlib_buffer_t * p4, * p5;
90 vlib_buffer_t * p6, * p7;
92 p4 = vlib_get_buffer (vm, from[4]);
93 p5 = vlib_get_buffer (vm, from[5]);
94 p6 = vlib_get_buffer (vm, from[6]);
95 p7 = vlib_get_buffer (vm, from[7]);
97 vlib_prefetch_buffer_header (p4, LOAD);
98 vlib_prefetch_buffer_header (p5, LOAD);
99 vlib_prefetch_buffer_header (p6, LOAD);
100 vlib_prefetch_buffer_header (p7, LOAD);
117 b0 = vlib_get_buffer(vm, bi0);
118 b1 = vlib_get_buffer(vm, bi1);
119 b2 = vlib_get_buffer(vm, bi2);
120 b3 = vlib_get_buffer(vm, bi3);
122 /* Follow the DPO on which the midchain is stacked */
123 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
124 adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
125 adj_index2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
126 adj_index3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
128 adj0 = adj_get(adj_index0);
129 adj1 = adj_get(adj_index1);
130 adj2 = adj_get(adj_index2);
131 adj3 = adj_get(adj_index3);
133 dpo0 = &adj0->sub_type.midchain.next_dpo;
134 dpo1 = &adj1->sub_type.midchain.next_dpo;
135 dpo2 = &adj2->sub_type.midchain.next_dpo;
136 dpo3 = &adj3->sub_type.midchain.next_dpo;
138 next0 = dpo0->dpoi_next_node;
139 next1 = dpo1->dpoi_next_node;
140 next2 = dpo2->dpoi_next_node;
141 next3 = dpo3->dpoi_next_node;
143 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
144 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
145 vnet_buffer(b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
146 vnet_buffer(b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
150 vlib_increment_combined_counter (im->combined_sw_if_counters
151 + VNET_INTERFACE_COUNTER_TX,
153 adj0->rewrite_header.sw_if_index,
155 vlib_buffer_length_in_chain (vm, b0));
156 vlib_increment_combined_counter (im->combined_sw_if_counters
157 + VNET_INTERFACE_COUNTER_TX,
159 adj1->rewrite_header.sw_if_index,
161 vlib_buffer_length_in_chain (vm, b1));
162 vlib_increment_combined_counter (im->combined_sw_if_counters
163 + VNET_INTERFACE_COUNTER_TX,
165 adj2->rewrite_header.sw_if_index,
167 vlib_buffer_length_in_chain (vm, b2));
168 vlib_increment_combined_counter (im->combined_sw_if_counters
169 + VNET_INTERFACE_COUNTER_TX,
171 adj3->rewrite_header.sw_if_index,
173 vlib_buffer_length_in_chain (vm, b3));
176 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
178 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
182 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
184 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
188 if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
190 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
194 if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
196 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
201 vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
202 to_next, n_left_to_next,
204 next0, next1, next2, next3);
206 while (n_left_from > 0 && n_left_to_next > 0)
208 u32 bi0, adj_index0, next0;
209 const ip_adjacency_t * adj0;
210 const dpo_id_t *dpo0;
220 b0 = vlib_get_buffer(vm, bi0);
222 /* Follow the DPO on which the midchain is stacked */
223 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
224 adj0 = adj_get(adj_index0);
225 dpo0 = &adj0->sub_type.midchain.next_dpo;
226 next0 = dpo0->dpoi_next_node;
227 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
231 vlib_increment_combined_counter (im->combined_sw_if_counters
232 + VNET_INTERFACE_COUNTER_TX,
234 adj0->rewrite_header.sw_if_index,
236 vlib_buffer_length_in_chain (vm, b0));
239 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
241 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
246 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
247 to_next, n_left_to_next,
251 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
254 return frame->n_vectors;
258 format_adj_midchain_tx_trace (u8 * s, va_list * args)
260 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
261 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
262 adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
264 s = format(s, "adj-midchain:[%d]:%U", tr->ai,
265 format_ip_adjacency, tr->ai,
266 FORMAT_IP_ADJACENCY_NONE);
272 adj_midchain_tx (vlib_main_t * vm,
273 vlib_node_runtime_t * node,
274 vlib_frame_t * frame)
276 return (adj_midchain_tx_inline(vm, node, frame, 1));
279 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
280 .function = adj_midchain_tx,
281 .name = "adj-midchain-tx",
282 .vector_size = sizeof (u32),
284 .format_trace = format_adj_midchain_tx_trace,
293 adj_midchain_tx_no_count (vlib_main_t * vm,
294 vlib_node_runtime_t * node,
295 vlib_frame_t * frame)
297 return (adj_midchain_tx_inline(vm, node, frame, 0));
300 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
301 .function = adj_midchain_tx_no_count,
302 .name = "adj-midchain-tx-no-count",
303 .vector_size = sizeof (u32),
305 .format_trace = format_adj_midchain_tx_trace,
313 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
314 .arc_name = "ip4-output",
315 .node_name = "adj-midchain-tx",
316 .runs_before = VNET_FEATURES ("interface-output"),
317 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
319 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
320 .arc_name = "ip4-output",
321 .node_name = "adj-midchain-tx-no-count",
322 .runs_before = VNET_FEATURES ("interface-output"),
323 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
325 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
326 .arc_name = "ip6-output",
327 .node_name = "adj-midchain-tx",
328 .runs_before = VNET_FEATURES ("interface-output"),
329 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
331 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
332 .arc_name = "ip6-output",
333 .node_name = "adj-midchain-tx-no-count",
334 .runs_before = VNET_FEATURES ("interface-output"),
335 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
337 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
338 .arc_name = "mpls-output",
339 .node_name = "adj-midchain-tx",
340 .runs_before = VNET_FEATURES ("interface-output"),
341 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
343 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
344 .arc_name = "mpls-output",
345 .node_name = "adj-midchain-tx-no-count",
346 .runs_before = VNET_FEATURES ("interface-output"),
347 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
349 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
350 .arc_name = "ethernet-output",
351 .node_name = "adj-midchain-tx",
352 .runs_before = VNET_FEATURES ("error-drop"),
353 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
355 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
356 .arc_name = "ethernet-output",
357 .node_name = "adj-midchain-tx-no-count",
358 .runs_before = VNET_FEATURES ("error-drop"),
359 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
361 VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = {
362 .arc_name = "nsh-output",
363 .node_name = "adj-midchain-tx",
364 .runs_before = VNET_FEATURES ("error-drop"),
365 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH],
367 VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = {
368 .arc_name = "nsh-output",
369 .node_name = "adj-midchain-tx-no-count",
370 .runs_before = VNET_FEATURES ("error-drop"),
371 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH],
375 adj_get_midchain_node (vnet_link_t link)
379 return (ip4_midchain_node.index);
381 return (ip6_midchain_node.index);
383 return (mpls_midchain_node.index);
384 case VNET_LINK_ETHERNET:
385 return (adj_l2_midchain_node.index);
387 return (adj_nsh_midchain_node.index);
396 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
399 switch (adj->ia_link)
403 arc = ip4_main.lookup_main.output_feature_arc_index;
408 arc = ip6_main.lookup_main.output_feature_arc_index;
413 arc = mpls_main.output_feature_arc_index;
416 case VNET_LINK_ETHERNET:
418 arc = ethernet_main.output_feature_arc_index;
423 arc = nsh_main_dummy.output_feature_arc_index;
431 ASSERT (arc != (u8) ~0);
437 adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
439 return ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
440 adj_midchain_tx_no_count_node.index :
441 adj_midchain_tx_node.index);
445 adj_nbr_midchain_get_feature_node (ip_adjacency_t *adj)
447 if (adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT)
449 return (adj_midchain_tx_no_count_feature_node[adj->ia_link]);
452 return (adj_midchain_tx_feature_node[adj->ia_link]);
458 * Setup the adj as a mid-chain
461 adj_midchain_setup (adj_index_t adj_index,
462 adj_midchain_fixup_t fixup,
466 u32 feature_index, tx_node;
470 ASSERT(ADJ_INDEX_INVALID != adj_index);
472 adj = adj_get(adj_index);
474 adj->sub_type.midchain.fixup_func = fixup;
475 adj->sub_type.midchain.fixup_data = data;
476 adj->ia_flags |= flags;
478 arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
479 feature_index = adj_nbr_midchain_get_feature_node(adj);
480 tx_node = adj_nbr_midchain_get_tx_node(adj);
482 vnet_feature_enable_disable_with_index (arc_index, feature_index,
483 adj->rewrite_header.sw_if_index,
484 1 /* enable */, 0, 0);
487 * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
488 * The graph arc used/created here is from the midchain-tx node to the
489 * child's registered node. This is because post adj processing the next
490 * node are any output features, then the midchain-tx. from there we
491 * need to get to the stacked child's node.
493 dpo_stack_from_node(tx_node,
494 &adj->sub_type.midchain.next_dpo,
495 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
499 * adj_nbr_midchain_update_rewrite
501 * Update the adjacency's rewrite string. A NULL string implies the
502 * rewrite is reset (i.e. when ARP/ND etnry is gone).
503 * NB: the adj being updated may be handling traffic in the DP.
506 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
507 adj_midchain_fixup_t fixup,
508 const void *fixup_data,
514 ASSERT(ADJ_INDEX_INVALID != adj_index);
516 adj = adj_get(adj_index);
519 * one time only update. since we don't support chainging the tunnel
520 * src,dst, this is all we need.
522 ASSERT((adj->lookup_next_index == IP_LOOKUP_NEXT_ARP) ||
523 (adj->lookup_next_index == IP_LOOKUP_NEXT_GLEAN) ||
524 (adj->lookup_next_index == IP_LOOKUP_NEXT_BCAST));
527 * tunnels can always provide a rewrite.
529 ASSERT(NULL != rewrite);
531 adj_midchain_setup(adj_index, fixup, fixup_data, flags);
534 * update the rewirte with the workers paused.
536 adj_nbr_update_rewrite_internal(adj,
537 IP_LOOKUP_NEXT_MIDCHAIN,
538 adj_get_midchain_node(adj->ia_link),
539 adj_nbr_midchain_get_tx_node(adj),
544 * adj_nbr_midchain_unstack
546 * Unstack the adj. stack it on drop
549 adj_nbr_midchain_unstack (adj_index_t adj_index)
553 ASSERT(ADJ_INDEX_INVALID != adj_index);
555 adj = adj_get(adj_index);
560 dpo_stack(DPO_ADJACENCY_MIDCHAIN,
561 vnet_link_to_dpo_proto(adj->ia_link),
562 &adj->sub_type.midchain.next_dpo,
563 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
564 CLIB_MEMORY_BARRIER();
568 * adj_nbr_midchain_stack
571 adj_nbr_midchain_stack (adj_index_t adj_index,
572 const dpo_id_t *next)
576 ASSERT(ADJ_INDEX_INVALID != adj_index);
578 adj = adj_get(adj_index);
580 ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) ||
581 (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index));
583 dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
584 &adj->sub_type.midchain.next_dpo,
589 format_adj_midchain (u8* s, va_list *ap)
591 index_t index = va_arg(*ap, index_t);
592 u32 indent = va_arg(*ap, u32);
593 ip_adjacency_t * adj = adj_get(index);
595 s = format (s, "%U", format_vnet_link, adj->ia_link);
596 s = format (s, " via %U",
597 format_ip46_address, &adj->sub_type.nbr.next_hop,
598 adj_proto_to_46(adj->ia_nh_proto));
599 s = format (s, " %U",
601 &adj->rewrite_header, sizeof (adj->rewrite_data), indent);
602 s = format (s, "\n%Ustacked-on:\n%U%U",
603 format_white_space, indent,
604 format_white_space, indent+2,
605 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
611 adj_dpo_lock (dpo_id_t *dpo)
613 adj_lock(dpo->dpoi_index);
616 adj_dpo_unlock (dpo_id_t *dpo)
618 adj_unlock(dpo->dpoi_index);
621 const static dpo_vft_t adj_midchain_dpo_vft = {
622 .dv_lock = adj_dpo_lock,
623 .dv_unlock = adj_dpo_unlock,
624 .dv_format = format_adj_midchain,
625 .dv_get_urpf = adj_dpo_get_urpf,
629 * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
632 * this means that these graph nodes are ones from which a midchain is the
633 * parent object in the DPO-graph.
635 const static char* const midchain_ip4_nodes[] =
640 const static char* const midchain_ip6_nodes[] =
645 const static char* const midchain_mpls_nodes[] =
650 const static char* const midchain_ethernet_nodes[] =
655 const static char* const midchain_nsh_nodes[] =
661 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
663 [DPO_PROTO_IP4] = midchain_ip4_nodes,
664 [DPO_PROTO_IP6] = midchain_ip6_nodes,
665 [DPO_PROTO_MPLS] = midchain_mpls_nodes,
666 [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
667 [DPO_PROTO_NSH] = midchain_nsh_nodes,
671 adj_midchain_module_init (void)
673 dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);