2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/adj/adj_nbr.h>
17 #include <vnet/adj/adj_internal.h>
18 #include <vnet/adj/adj_l2.h>
19 #include <vnet/adj/adj_nsh.h>
20 #include <vnet/adj/adj_midchain.h>
21 #include <vnet/ethernet/arp_packet.h>
22 #include <vnet/dpo/drop_dpo.h>
23 #include <vnet/fib/fib_walk.h>
26 * The two midchain tx feature node indices
28 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
29 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
32 * @brief Trace data for packets traversing the midchain tx node
34 typedef struct adj_midchain_tx_trace_t_
37 * @brief the midchain adj we are traversing
40 } adj_midchain_tx_trace_t;
43 adj_midchain_tx_inline (vlib_main_t * vm,
44 vlib_node_runtime_t * node,
48 u32 * from, * to_next, n_left_from, n_left_to_next;
50 vnet_main_t *vnm = vnet_get_main ();
51 vnet_interface_main_t *im = &vnm->interface_main;
52 u32 cpu_index = vm->cpu_index;
54 /* Vector of buffer / pkt indices we're supposed to process */
55 from = vlib_frame_vector_args (frame);
57 /* Number of buffers / pkts */
58 n_left_from = frame->n_vectors;
60 /* Speculatively send the first buffer to the last disposition we used */
61 next_index = node->cached_next_index;
63 while (n_left_from > 0)
65 /* set up to enqueue to our disposition with index = next_index */
66 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
69 while (n_left_from >= 4 && n_left_to_next > 2)
71 u32 bi0, adj_index0, next0;
72 const ip_adjacency_t * adj0;
75 u32 bi1, adj_index1, next1;
76 const ip_adjacency_t * adj1;
80 /* Prefetch next iteration. */
82 vlib_buffer_t * p2, * p3;
84 p2 = vlib_get_buffer (vm, from[2]);
85 p3 = vlib_get_buffer (vm, from[3]);
87 vlib_prefetch_buffer_header (p2, LOAD);
88 vlib_prefetch_buffer_header (p3, LOAD);
90 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
91 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
104 b0 = vlib_get_buffer(vm, bi0);
105 b1 = vlib_get_buffer(vm, bi1);
107 /* Follow the DPO on which the midchain is stacked */
108 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
109 adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
111 adj0 = adj_get(adj_index0);
112 adj1 = adj_get(adj_index1);
114 dpo0 = &adj0->sub_type.midchain.next_dpo;
115 dpo1 = &adj1->sub_type.midchain.next_dpo;
117 next0 = dpo0->dpoi_next_node;
118 next1 = dpo1->dpoi_next_node;
120 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
121 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
125 vlib_increment_combined_counter (im->combined_sw_if_counters
126 + VNET_INTERFACE_COUNTER_TX,
128 adj0->rewrite_header.sw_if_index,
130 vlib_buffer_length_in_chain (vm, b0));
131 vlib_increment_combined_counter (im->combined_sw_if_counters
132 + VNET_INTERFACE_COUNTER_TX,
134 adj1->rewrite_header.sw_if_index,
136 vlib_buffer_length_in_chain (vm, b1));
139 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
141 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
145 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
147 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
152 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
153 to_next, n_left_to_next,
157 while (n_left_from > 0 && n_left_to_next > 0)
159 u32 bi0, adj_index0, next0;
160 const ip_adjacency_t * adj0;
161 const dpo_id_t *dpo0;
171 b0 = vlib_get_buffer(vm, bi0);
173 /* Follow the DPO on which the midchain is stacked */
174 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
175 adj0 = adj_get(adj_index0);
176 dpo0 = &adj0->sub_type.midchain.next_dpo;
177 next0 = dpo0->dpoi_next_node;
178 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
182 vlib_increment_combined_counter (im->combined_sw_if_counters
183 + VNET_INTERFACE_COUNTER_TX,
185 adj0->rewrite_header.sw_if_index,
187 vlib_buffer_length_in_chain (vm, b0));
190 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
192 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
197 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
198 to_next, n_left_to_next,
202 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
205 return frame->n_vectors;
209 format_adj_midchain_tx_trace (u8 * s, va_list * args)
211 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
212 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
213 adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
215 s = format(s, "adj-midchain:[%d]:%U", tr->ai,
216 format_ip_adjacency, tr->ai,
217 FORMAT_IP_ADJACENCY_NONE);
223 adj_midchain_tx (vlib_main_t * vm,
224 vlib_node_runtime_t * node,
225 vlib_frame_t * frame)
227 return (adj_midchain_tx_inline(vm, node, frame, 1));
230 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
231 .function = adj_midchain_tx,
232 .name = "adj-midchain-tx",
233 .vector_size = sizeof (u32),
235 .format_trace = format_adj_midchain_tx_trace,
244 adj_midchain_tx_no_count (vlib_main_t * vm,
245 vlib_node_runtime_t * node,
246 vlib_frame_t * frame)
248 return (adj_midchain_tx_inline(vm, node, frame, 0));
251 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
252 .function = adj_midchain_tx_no_count,
253 .name = "adj-midchain-tx-no-count",
254 .vector_size = sizeof (u32),
256 .format_trace = format_adj_midchain_tx_trace,
264 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
265 .arc_name = "ip4-output",
266 .node_name = "adj-midchain-tx",
267 .runs_before = VNET_FEATURES ("interface-output"),
268 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
270 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
271 .arc_name = "ip4-output",
272 .node_name = "adj-midchain-tx-no-count",
273 .runs_before = VNET_FEATURES ("interface-output"),
274 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
276 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
277 .arc_name = "ip6-output",
278 .node_name = "adj-midchain-tx",
279 .runs_before = VNET_FEATURES ("interface-output"),
280 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
282 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
283 .arc_name = "ip6-output",
284 .node_name = "adj-midchain-tx-no-count",
285 .runs_before = VNET_FEATURES ("interface-output"),
286 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
288 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
289 .arc_name = "mpls-output",
290 .node_name = "adj-midchain-tx",
291 .runs_before = VNET_FEATURES ("interface-output"),
292 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
294 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
295 .arc_name = "mpls-output",
296 .node_name = "adj-midchain-tx-no-count",
297 .runs_before = VNET_FEATURES ("interface-output"),
298 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
300 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
301 .arc_name = "ethernet-output",
302 .node_name = "adj-midchain-tx",
303 .runs_before = VNET_FEATURES ("error-drop"),
304 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
306 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
307 .arc_name = "ethernet-output",
308 .node_name = "adj-midchain-tx-no-count",
309 .runs_before = VNET_FEATURES ("error-drop"),
310 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
312 VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = {
313 .arc_name = "nsh-output",
314 .node_name = "adj-midchain-tx",
315 .runs_before = VNET_FEATURES ("error-drop"),
316 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH],
318 VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = {
319 .arc_name = "nsh-output",
320 .node_name = "adj-midchain-tx-no-count",
321 .runs_before = VNET_FEATURES ("error-drop"),
322 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH],
326 adj_get_midchain_node (vnet_link_t link)
330 return (ip4_midchain_node.index);
332 return (ip6_midchain_node.index);
334 return (mpls_midchain_node.index);
335 case VNET_LINK_ETHERNET:
336 return (adj_l2_midchain_node.index);
338 return (adj_nsh_midchain_node.index);
347 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
350 switch (adj->ia_link)
354 arc = ip4_main.lookup_main.output_feature_arc_index;
359 arc = ip6_main.lookup_main.output_feature_arc_index;
364 arc = mpls_main.output_feature_arc_index;
367 case VNET_LINK_ETHERNET:
369 arc = ethernet_main.output_feature_arc_index;
374 arc = nsh_main_dummy.output_feature_arc_index;
382 ASSERT (arc != (u8) ~0);
388 * adj_nbr_midchain_update_rewrite
390 * Update the adjacency's rewrite string. A NULL string implies the
391 * rewrite is reset (i.e. when ARP/ND etnry is gone).
392 * NB: the adj being updated may be handling traffic in the DP.
395 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
396 adj_midchain_fixup_t fixup,
397 adj_midchain_flag_t flags,
404 ASSERT(ADJ_INDEX_INVALID != adj_index);
406 adj = adj_get(adj_index);
409 * one time only update. since we don't support chainging the tunnel
410 * src,dst, this is all we need.
412 ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP);
414 * tunnels can always provide a rewrite.
416 ASSERT(NULL != rewrite);
418 adj->sub_type.midchain.fixup_func = fixup;
420 arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
421 feature_index = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
422 adj_midchain_tx_no_count_feature_node[adj->ia_link] :
423 adj_midchain_tx_feature_node[adj->ia_link];
425 adj->sub_type.midchain.tx_function_node = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
426 adj_midchain_tx_no_count_node.index :
427 adj_midchain_tx_node.index;
429 vnet_feature_enable_disable_with_index (arc_index, feature_index,
430 adj->rewrite_header.sw_if_index,
431 1 /* enable */, 0, 0);
434 * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
435 * The graph arc used/created here is from the midchain-tx node to the
436 * child's registered node. This is because post adj processing the next
437 * node are any output features, then the midchain-tx. from there we
438 * need to get to the stacked child's node.
440 dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
441 &adj->sub_type.midchain.next_dpo,
442 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
445 * update the rewirte with the workers paused.
447 adj_nbr_update_rewrite_internal(adj,
448 IP_LOOKUP_NEXT_MIDCHAIN,
449 adj_get_midchain_node(adj->ia_link),
450 adj->sub_type.midchain.tx_function_node,
455 * adj_nbr_midchain_unstack
457 * Unstack the adj. stack it on drop
460 adj_nbr_midchain_unstack (adj_index_t adj_index)
464 ASSERT(ADJ_INDEX_INVALID != adj_index);
466 adj = adj_get(adj_index);
471 dpo_stack(DPO_ADJACENCY_MIDCHAIN,
472 vnet_link_to_dpo_proto(adj->ia_link),
473 &adj->sub_type.midchain.next_dpo,
474 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
476 CLIB_MEMORY_BARRIER();
480 * adj_nbr_midchain_stack
483 adj_nbr_midchain_stack (adj_index_t adj_index,
484 const dpo_id_t *next)
488 ASSERT(ADJ_INDEX_INVALID != adj_index);
490 adj = adj_get(adj_index);
492 ASSERT(IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index);
494 dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
495 &adj->sub_type.midchain.next_dpo,
500 format_adj_midchain (u8* s, va_list *ap)
502 index_t index = va_arg(*ap, index_t);
503 u32 indent = va_arg(*ap, u32);
504 vnet_main_t * vnm = vnet_get_main();
505 ip_adjacency_t * adj = adj_get(index);
507 s = format (s, "%U", format_vnet_link, adj->ia_link);
508 s = format (s, " via %U ",
509 format_ip46_address, &adj->sub_type.nbr.next_hop);
510 s = format (s, " %U",
512 vnm->vlib_main, &adj->rewrite_header,
513 sizeof (adj->rewrite_data), indent);
514 s = format (s, "\n%Ustacked-on:\n%U%U",
515 format_white_space, indent,
516 format_white_space, indent+2,
517 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
523 adj_dpo_lock (dpo_id_t *dpo)
525 adj_lock(dpo->dpoi_index);
528 adj_dpo_unlock (dpo_id_t *dpo)
530 adj_unlock(dpo->dpoi_index);
533 const static dpo_vft_t adj_midchain_dpo_vft = {
534 .dv_lock = adj_dpo_lock,
535 .dv_unlock = adj_dpo_unlock,
536 .dv_format = format_adj_midchain,
540 * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
543 * this means that these graph nodes are ones from which a midchain is the
544 * parent object in the DPO-graph.
546 const static char* const midchain_ip4_nodes[] =
551 const static char* const midchain_ip6_nodes[] =
556 const static char* const midchain_mpls_nodes[] =
561 const static char* const midchain_ethernet_nodes[] =
567 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
569 [DPO_PROTO_IP4] = midchain_ip4_nodes,
570 [DPO_PROTO_IP6] = midchain_ip6_nodes,
571 [DPO_PROTO_MPLS] = midchain_mpls_nodes,
572 [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
576 adj_midchain_module_init (void)
578 dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);