2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/adj/adj_nbr.h>
17 #include <vnet/adj/adj_internal.h>
18 #include <vnet/adj/adj_l2.h>
19 #include <vnet/adj/adj_nsh.h>
20 #include <vnet/adj/adj_midchain.h>
21 #include <vnet/ethernet/arp_packet.h>
22 #include <vnet/dpo/drop_dpo.h>
23 #include <vnet/fib/fib_walk.h>
26 * The two midchain tx feature node indices
28 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
29 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
32 * @brief Trace data for packets traversing the midchain tx node
34 typedef struct adj_midchain_tx_trace_t_
37 * @brief the midchain adj we are traversing
40 } adj_midchain_tx_trace_t;
43 adj_midchain_tx_inline (vlib_main_t * vm,
44 vlib_node_runtime_t * node,
48 u32 * from, * to_next, n_left_from, n_left_to_next;
50 vnet_main_t *vnm = vnet_get_main ();
51 vnet_interface_main_t *im = &vnm->interface_main;
52 u32 thread_index = vm->thread_index;
54 /* Vector of buffer / pkt indices we're supposed to process */
55 from = vlib_frame_vector_args (frame);
57 /* Number of buffers / pkts */
58 n_left_from = frame->n_vectors;
60 /* Speculatively send the first buffer to the last disposition we used */
61 next_index = node->cached_next_index;
63 while (n_left_from > 0)
65 /* set up to enqueue to our disposition with index = next_index */
66 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
69 while (n_left_from >= 4 && n_left_to_next > 2)
71 u32 bi0, adj_index0, next0;
72 const ip_adjacency_t * adj0;
75 u32 bi1, adj_index1, next1;
76 const ip_adjacency_t * adj1;
80 /* Prefetch next iteration. */
82 vlib_buffer_t * p2, * p3;
84 p2 = vlib_get_buffer (vm, from[2]);
85 p3 = vlib_get_buffer (vm, from[3]);
87 vlib_prefetch_buffer_header (p2, LOAD);
88 vlib_prefetch_buffer_header (p3, LOAD);
90 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
91 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
104 b0 = vlib_get_buffer(vm, bi0);
105 b1 = vlib_get_buffer(vm, bi1);
107 /* Follow the DPO on which the midchain is stacked */
108 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
109 adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
111 adj0 = adj_get(adj_index0);
112 adj1 = adj_get(adj_index1);
114 dpo0 = &adj0->sub_type.midchain.next_dpo;
115 dpo1 = &adj1->sub_type.midchain.next_dpo;
117 next0 = dpo0->dpoi_next_node;
118 next1 = dpo1->dpoi_next_node;
120 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
121 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
125 vlib_increment_combined_counter (im->combined_sw_if_counters
126 + VNET_INTERFACE_COUNTER_TX,
128 adj0->rewrite_header.sw_if_index,
130 vlib_buffer_length_in_chain (vm, b0));
131 vlib_increment_combined_counter (im->combined_sw_if_counters
132 + VNET_INTERFACE_COUNTER_TX,
134 adj1->rewrite_header.sw_if_index,
136 vlib_buffer_length_in_chain (vm, b1));
139 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
141 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
145 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
147 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
152 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
153 to_next, n_left_to_next,
157 while (n_left_from > 0 && n_left_to_next > 0)
159 u32 bi0, adj_index0, next0;
160 const ip_adjacency_t * adj0;
161 const dpo_id_t *dpo0;
171 b0 = vlib_get_buffer(vm, bi0);
173 /* Follow the DPO on which the midchain is stacked */
174 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
175 adj0 = adj_get(adj_index0);
176 dpo0 = &adj0->sub_type.midchain.next_dpo;
177 next0 = dpo0->dpoi_next_node;
178 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
182 vlib_increment_combined_counter (im->combined_sw_if_counters
183 + VNET_INTERFACE_COUNTER_TX,
185 adj0->rewrite_header.sw_if_index,
187 vlib_buffer_length_in_chain (vm, b0));
190 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
192 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
197 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
198 to_next, n_left_to_next,
202 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
205 return frame->n_vectors;
209 format_adj_midchain_tx_trace (u8 * s, va_list * args)
211 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
212 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
213 adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
215 s = format(s, "adj-midchain:[%d]:%U", tr->ai,
216 format_ip_adjacency, tr->ai,
217 FORMAT_IP_ADJACENCY_NONE);
223 adj_midchain_tx (vlib_main_t * vm,
224 vlib_node_runtime_t * node,
225 vlib_frame_t * frame)
227 return (adj_midchain_tx_inline(vm, node, frame, 1));
230 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
231 .function = adj_midchain_tx,
232 .name = "adj-midchain-tx",
233 .vector_size = sizeof (u32),
235 .format_trace = format_adj_midchain_tx_trace,
244 adj_midchain_tx_no_count (vlib_main_t * vm,
245 vlib_node_runtime_t * node,
246 vlib_frame_t * frame)
248 return (adj_midchain_tx_inline(vm, node, frame, 0));
251 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
252 .function = adj_midchain_tx_no_count,
253 .name = "adj-midchain-tx-no-count",
254 .vector_size = sizeof (u32),
256 .format_trace = format_adj_midchain_tx_trace,
264 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
265 .arc_name = "ip4-output",
266 .node_name = "adj-midchain-tx",
267 .runs_before = VNET_FEATURES ("interface-output"),
268 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
270 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
271 .arc_name = "ip4-output",
272 .node_name = "adj-midchain-tx-no-count",
273 .runs_before = VNET_FEATURES ("interface-output"),
274 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
276 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
277 .arc_name = "ip6-output",
278 .node_name = "adj-midchain-tx",
279 .runs_before = VNET_FEATURES ("interface-output"),
280 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
282 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
283 .arc_name = "ip6-output",
284 .node_name = "adj-midchain-tx-no-count",
285 .runs_before = VNET_FEATURES ("interface-output"),
286 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
288 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
289 .arc_name = "mpls-output",
290 .node_name = "adj-midchain-tx",
291 .runs_before = VNET_FEATURES ("interface-output"),
292 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
294 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
295 .arc_name = "mpls-output",
296 .node_name = "adj-midchain-tx-no-count",
297 .runs_before = VNET_FEATURES ("interface-output"),
298 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
300 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
301 .arc_name = "ethernet-output",
302 .node_name = "adj-midchain-tx",
303 .runs_before = VNET_FEATURES ("error-drop"),
304 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
306 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
307 .arc_name = "ethernet-output",
308 .node_name = "adj-midchain-tx-no-count",
309 .runs_before = VNET_FEATURES ("error-drop"),
310 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
312 VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = {
313 .arc_name = "nsh-output",
314 .node_name = "adj-midchain-tx",
315 .runs_before = VNET_FEATURES ("error-drop"),
316 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH],
318 VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = {
319 .arc_name = "nsh-output",
320 .node_name = "adj-midchain-tx-no-count",
321 .runs_before = VNET_FEATURES ("error-drop"),
322 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH],
326 adj_get_midchain_node (vnet_link_t link)
330 return (ip4_midchain_node.index);
332 return (ip6_midchain_node.index);
334 return (mpls_midchain_node.index);
335 case VNET_LINK_ETHERNET:
336 return (adj_l2_midchain_node.index);
338 return (adj_nsh_midchain_node.index);
347 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
350 switch (adj->ia_link)
354 arc = ip4_main.lookup_main.output_feature_arc_index;
359 arc = ip6_main.lookup_main.output_feature_arc_index;
364 arc = mpls_main.output_feature_arc_index;
367 case VNET_LINK_ETHERNET:
369 arc = ethernet_main.output_feature_arc_index;
374 arc = nsh_main_dummy.output_feature_arc_index;
382 ASSERT (arc != (u8) ~0);
388 adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
390 return ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
391 adj_midchain_tx_no_count_node.index :
392 adj_midchain_tx_node.index);
398 * Setup the adj as a mid-chain
401 adj_midchain_setup (adj_index_t adj_index,
402 adj_midchain_fixup_t fixup,
405 u32 feature_index, tx_node;
409 ASSERT(ADJ_INDEX_INVALID != adj_index);
411 adj = adj_get(adj_index);
413 adj->sub_type.midchain.fixup_func = fixup;
414 adj->ia_flags |= flags;
416 arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
417 feature_index = (flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
418 adj_midchain_tx_no_count_feature_node[adj->ia_link] :
419 adj_midchain_tx_feature_node[adj->ia_link];
421 tx_node = adj_nbr_midchain_get_tx_node(adj);
423 vnet_feature_enable_disable_with_index (arc_index, feature_index,
424 adj->rewrite_header.sw_if_index,
425 1 /* enable */, 0, 0);
428 * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
429 * The graph arc used/created here is from the midchain-tx node to the
430 * child's registered node. This is because post adj processing the next
431 * node are any output features, then the midchain-tx. from there we
432 * need to get to the stacked child's node.
434 dpo_stack_from_node(tx_node,
435 &adj->sub_type.midchain.next_dpo,
436 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
440 * adj_nbr_midchain_update_rewrite
442 * Update the adjacency's rewrite string. A NULL string implies the
443 * rewrite is reset (i.e. when ARP/ND etnry is gone).
444 * NB: the adj being updated may be handling traffic in the DP.
447 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
448 adj_midchain_fixup_t fixup,
454 ASSERT(ADJ_INDEX_INVALID != adj_index);
456 adj = adj_get(adj_index);
459 * one time only update. since we don't support chainging the tunnel
460 * src,dst, this is all we need.
462 ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP);
464 * tunnels can always provide a rewrite.
466 ASSERT(NULL != rewrite);
468 adj_midchain_setup(adj_index, fixup, flags);
471 * update the rewirte with the workers paused.
473 adj_nbr_update_rewrite_internal(adj,
474 IP_LOOKUP_NEXT_MIDCHAIN,
475 adj_get_midchain_node(adj->ia_link),
476 adj_nbr_midchain_get_tx_node(adj),
481 * adj_nbr_midchain_unstack
483 * Unstack the adj. stack it on drop
486 adj_nbr_midchain_unstack (adj_index_t adj_index)
490 ASSERT(ADJ_INDEX_INVALID != adj_index);
492 adj = adj_get(adj_index);
497 dpo_stack(DPO_ADJACENCY_MIDCHAIN,
498 vnet_link_to_dpo_proto(adj->ia_link),
499 &adj->sub_type.midchain.next_dpo,
500 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
502 CLIB_MEMORY_BARRIER();
506 * adj_nbr_midchain_stack
509 adj_nbr_midchain_stack (adj_index_t adj_index,
510 const dpo_id_t *next)
514 ASSERT(ADJ_INDEX_INVALID != adj_index);
516 adj = adj_get(adj_index);
518 ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) ||
519 (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index));
521 dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
522 &adj->sub_type.midchain.next_dpo,
527 format_adj_midchain (u8* s, va_list *ap)
529 index_t index = va_arg(*ap, index_t);
530 u32 indent = va_arg(*ap, u32);
531 ip_adjacency_t * adj = adj_get(index);
533 s = format (s, "%U", format_vnet_link, adj->ia_link);
534 s = format (s, " via %U ",
535 format_ip46_address, &adj->sub_type.nbr.next_hop);
536 s = format (s, " %U",
538 &adj->rewrite_header, sizeof (adj->rewrite_data), indent);
539 s = format (s, "\n%Ustacked-on:\n%U%U",
540 format_white_space, indent,
541 format_white_space, indent+2,
542 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
548 adj_dpo_lock (dpo_id_t *dpo)
550 adj_lock(dpo->dpoi_index);
553 adj_dpo_unlock (dpo_id_t *dpo)
555 adj_unlock(dpo->dpoi_index);
558 const static dpo_vft_t adj_midchain_dpo_vft = {
559 .dv_lock = adj_dpo_lock,
560 .dv_unlock = adj_dpo_unlock,
561 .dv_format = format_adj_midchain,
565 * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
568 * this means that these graph nodes are ones from which a midchain is the
569 * parent object in the DPO-graph.
571 const static char* const midchain_ip4_nodes[] =
576 const static char* const midchain_ip6_nodes[] =
581 const static char* const midchain_mpls_nodes[] =
586 const static char* const midchain_ethernet_nodes[] =
591 const static char* const midchain_nsh_nodes[] =
597 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
599 [DPO_PROTO_IP4] = midchain_ip4_nodes,
600 [DPO_PROTO_IP6] = midchain_ip6_nodes,
601 [DPO_PROTO_MPLS] = midchain_mpls_nodes,
602 [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
603 [DPO_PROTO_NSH] = midchain_nsh_nodes,
607 adj_midchain_module_init (void)
609 dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);