2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/adj/adj_nbr.h>
17 #include <vnet/adj/adj_internal.h>
18 #include <vnet/adj/adj_l2.h>
19 #include <vnet/adj/adj_midchain.h>
20 #include <vnet/ethernet/arp_packet.h>
21 #include <vnet/dpo/drop_dpo.h>
22 #include <vnet/fib/fib_walk.h>
25 * The two midchain tx feature node indices
27 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
28 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
31 * @brief Trace data for packets traversing the midchain tx node
33 typedef struct adj_midchain_tx_trace_t_
36 * @brief the midchain adj we are traversing
39 } adj_midchain_tx_trace_t;
42 adj_midchain_tx_inline (vlib_main_t * vm,
43 vlib_node_runtime_t * node,
47 u32 * from, * to_next, n_left_from, n_left_to_next;
49 vnet_main_t *vnm = vnet_get_main ();
50 vnet_interface_main_t *im = &vnm->interface_main;
51 u32 cpu_index = vm->cpu_index;
53 /* Vector of buffer / pkt indices we're supposed to process */
54 from = vlib_frame_vector_args (frame);
56 /* Number of buffers / pkts */
57 n_left_from = frame->n_vectors;
59 /* Speculatively send the first buffer to the last disposition we used */
60 next_index = node->cached_next_index;
62 while (n_left_from > 0)
64 /* set up to enqueue to our disposition with index = next_index */
65 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
68 while (n_left_from >= 4 && n_left_to_next > 2)
70 u32 bi0, adj_index0, next0;
71 const ip_adjacency_t * adj0;
74 u32 bi1, adj_index1, next1;
75 const ip_adjacency_t * adj1;
79 /* Prefetch next iteration. */
81 vlib_buffer_t * p2, * p3;
83 p2 = vlib_get_buffer (vm, from[2]);
84 p3 = vlib_get_buffer (vm, from[3]);
86 vlib_prefetch_buffer_header (p2, LOAD);
87 vlib_prefetch_buffer_header (p3, LOAD);
89 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
90 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
103 b0 = vlib_get_buffer(vm, bi0);
104 b1 = vlib_get_buffer(vm, bi1);
106 /* Follow the DPO on which the midchain is stacked */
107 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
108 adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
110 adj0 = adj_get(adj_index0);
111 adj1 = adj_get(adj_index1);
113 dpo0 = &adj0->sub_type.midchain.next_dpo;
114 dpo1 = &adj1->sub_type.midchain.next_dpo;
116 next0 = dpo0->dpoi_next_node;
117 next1 = dpo1->dpoi_next_node;
119 vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
120 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
124 vlib_increment_combined_counter (im->combined_sw_if_counters
125 + VNET_INTERFACE_COUNTER_TX,
127 adj0->rewrite_header.sw_if_index,
129 vlib_buffer_length_in_chain (vm, b0));
130 vlib_increment_combined_counter (im->combined_sw_if_counters
131 + VNET_INTERFACE_COUNTER_TX,
133 adj1->rewrite_header.sw_if_index,
135 vlib_buffer_length_in_chain (vm, b1));
138 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
140 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
144 if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
146 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
151 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
152 to_next, n_left_to_next,
156 while (n_left_from > 0 && n_left_to_next > 0)
158 u32 bi0, adj_index0, next0;
159 const ip_adjacency_t * adj0;
160 const dpo_id_t *dpo0;
170 b0 = vlib_get_buffer(vm, bi0);
172 /* Follow the DPO on which the midchain is stacked */
173 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
174 adj0 = adj_get(adj_index0);
175 dpo0 = &adj0->sub_type.midchain.next_dpo;
176 next0 = dpo0->dpoi_next_node;
177 vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
181 vlib_increment_combined_counter (im->combined_sw_if_counters
182 + VNET_INTERFACE_COUNTER_TX,
184 adj0->rewrite_header.sw_if_index,
186 vlib_buffer_length_in_chain (vm, b0));
189 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
191 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
196 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
197 to_next, n_left_to_next,
201 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
204 return frame->n_vectors;
208 format_adj_midchain_tx_trace (u8 * s, va_list * args)
210 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
211 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
212 adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
214 s = format(s, "adj-midchain:[%d]:%U", tr->ai,
215 format_ip_adjacency, tr->ai,
216 FORMAT_IP_ADJACENCY_NONE);
222 adj_midchain_tx (vlib_main_t * vm,
223 vlib_node_runtime_t * node,
224 vlib_frame_t * frame)
226 return (adj_midchain_tx_inline(vm, node, frame, 1));
229 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
230 .function = adj_midchain_tx,
231 .name = "adj-midchain-tx",
232 .vector_size = sizeof (u32),
234 .format_trace = format_adj_midchain_tx_trace,
243 adj_midchain_tx_no_count (vlib_main_t * vm,
244 vlib_node_runtime_t * node,
245 vlib_frame_t * frame)
247 return (adj_midchain_tx_inline(vm, node, frame, 0));
250 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
251 .function = adj_midchain_tx_no_count,
252 .name = "adj-midchain-tx-no-count",
253 .vector_size = sizeof (u32),
255 .format_trace = format_adj_midchain_tx_trace,
263 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
264 .arc_name = "ip4-output",
265 .node_name = "adj-midchain-tx",
266 .runs_before = VNET_FEATURES ("interface-output"),
267 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
269 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
270 .arc_name = "ip4-output",
271 .node_name = "adj-midchain-tx-no-count",
272 .runs_before = VNET_FEATURES ("interface-output"),
273 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
275 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
276 .arc_name = "ip6-output",
277 .node_name = "adj-midchain-tx",
278 .runs_before = VNET_FEATURES ("interface-output"),
279 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
281 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
282 .arc_name = "ip6-output",
283 .node_name = "adj-midchain-tx-no-count",
284 .runs_before = VNET_FEATURES ("interface-output"),
285 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
287 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
288 .arc_name = "mpls-output",
289 .node_name = "adj-midchain-tx",
290 .runs_before = VNET_FEATURES ("interface-output"),
291 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
293 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
294 .arc_name = "mpls-output",
295 .node_name = "adj-midchain-tx-no-count",
296 .runs_before = VNET_FEATURES ("interface-output"),
297 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
299 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
300 .arc_name = "ethernet-output",
301 .node_name = "adj-midchain-tx",
302 .runs_before = VNET_FEATURES ("error-drop"),
303 .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
305 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
306 .arc_name = "ethernet-output",
307 .node_name = "adj-midchain-tx-no-count",
308 .runs_before = VNET_FEATURES ("error-drop"),
309 .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
313 adj_get_midchain_node (vnet_link_t link)
317 return (ip4_midchain_node.index);
319 return (ip6_midchain_node.index);
321 return (mpls_midchain_node.index);
322 case VNET_LINK_ETHERNET:
323 return (adj_l2_midchain_node.index);
332 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
335 switch (adj->ia_link)
339 arc = ip4_main.lookup_main.output_feature_arc_index;
344 arc = ip6_main.lookup_main.output_feature_arc_index;
349 arc = mpls_main.output_feature_arc_index;
352 case VNET_LINK_ETHERNET:
354 arc = ethernet_main.output_feature_arc_index;
362 ASSERT (arc != (u8) ~0);
368 * adj_nbr_midchain_update_rewrite
370 * Update the adjacency's rewrite string. A NULL string implies the
371 * rewrite is reset (i.e. when ARP/ND etnry is gone).
372 * NB: the adj being updated may be handling traffic in the DP.
375 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
376 adj_midchain_fixup_t fixup,
377 adj_midchain_flag_t flags,
384 ASSERT(ADJ_INDEX_INVALID != adj_index);
386 adj = adj_get(adj_index);
389 * one time only update. since we don't support chainging the tunnel
390 * src,dst, this is all we need.
392 ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP);
394 * tunnels can always provide a rewrite.
396 ASSERT(NULL != rewrite);
398 adj->sub_type.midchain.fixup_func = fixup;
400 arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
401 feature_index = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
402 adj_midchain_tx_no_count_feature_node[adj->ia_link] :
403 adj_midchain_tx_feature_node[adj->ia_link];
405 adj->sub_type.midchain.tx_function_node = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
406 adj_midchain_tx_no_count_node.index :
407 adj_midchain_tx_node.index;
409 vnet_feature_enable_disable_with_index (arc_index, feature_index,
410 adj->rewrite_header.sw_if_index,
411 1 /* enable */, 0, 0);
414 * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
415 * The graph arc used/created here is from the midchain-tx node to the
416 * child's registered node. This is because post adj processing the next
417 * node are any output features, then the midchain-tx. from there we
418 * need to get to the stacked child's node.
420 dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
421 &adj->sub_type.midchain.next_dpo,
422 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
425 * update the rewirte with the workers paused.
427 adj_nbr_update_rewrite_internal(adj,
428 IP_LOOKUP_NEXT_MIDCHAIN,
429 adj_get_midchain_node(adj->ia_link),
430 adj->sub_type.midchain.tx_function_node,
435 * adj_nbr_midchain_unstack
437 * Unstack the adj. stack it on drop
440 adj_nbr_midchain_unstack (adj_index_t adj_index)
444 ASSERT(ADJ_INDEX_INVALID != adj_index);
446 adj = adj_get(adj_index);
451 dpo_stack(DPO_ADJACENCY_MIDCHAIN,
452 vnet_link_to_dpo_proto(adj->ia_link),
453 &adj->sub_type.midchain.next_dpo,
454 drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
456 CLIB_MEMORY_BARRIER();
460 * adj_nbr_midchain_stack
463 adj_nbr_midchain_stack (adj_index_t adj_index,
464 const dpo_id_t *next)
468 ASSERT(ADJ_INDEX_INVALID != adj_index);
470 adj = adj_get(adj_index);
472 ASSERT(IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index);
474 dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
475 &adj->sub_type.midchain.next_dpo,
480 format_adj_midchain (u8* s, va_list *ap)
482 index_t index = va_arg(*ap, index_t);
483 u32 indent = va_arg(*ap, u32);
484 vnet_main_t * vnm = vnet_get_main();
485 ip_adjacency_t * adj = adj_get(index);
487 s = format (s, "%U", format_vnet_link, adj->ia_link);
488 s = format (s, " via %U ",
489 format_ip46_address, &adj->sub_type.nbr.next_hop);
490 s = format (s, " %U",
492 vnm->vlib_main, &adj->rewrite_header,
493 sizeof (adj->rewrite_data), indent);
494 s = format (s, "\n%Ustacked-on:\n%U%U",
495 format_white_space, indent,
496 format_white_space, indent+2,
497 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
503 adj_dpo_lock (dpo_id_t *dpo)
505 adj_lock(dpo->dpoi_index);
508 adj_dpo_unlock (dpo_id_t *dpo)
510 adj_unlock(dpo->dpoi_index);
513 const static dpo_vft_t adj_midchain_dpo_vft = {
514 .dv_lock = adj_dpo_lock,
515 .dv_unlock = adj_dpo_unlock,
516 .dv_format = format_adj_midchain,
520 * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
523 * this means that these graph nodes are ones from which a midchain is the
524 * parent object in the DPO-graph.
526 const static char* const midchain_ip4_nodes[] =
531 const static char* const midchain_ip6_nodes[] =
536 const static char* const midchain_mpls_nodes[] =
541 const static char* const midchain_ethernet_nodes[] =
547 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
549 [DPO_PROTO_IP4] = midchain_ip4_nodes,
550 [DPO_PROTO_IP6] = midchain_ip6_nodes,
551 [DPO_PROTO_MPLS] = midchain_mpls_nodes,
552 [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
556 adj_midchain_module_init (void)
558 dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);