u32 next_index;
vnet_main_t *vnm = vnet_get_main ();
vnet_interface_main_t *im = &vnm->interface_main;
- u32 cpu_index = vm->cpu_index;
+ u32 thread_index = vm->thread_index;
/* Vector of buffer / pkt indices we're supposed to process */
from = vlib_frame_vector_args (frame);
/* set up to enqueue to our disposition with index = next_index */
vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-
- while (n_left_from >= 4 && n_left_to_next > 2)
+ while (n_left_from >= 8 && n_left_to_next > 4)
{
u32 bi0, adj_index0, next0;
const ip_adjacency_t * adj0;
const ip_adjacency_t * adj1;
const dpo_id_t *dpo1;
vlib_buffer_t * b1;
+ u32 bi2, adj_index2, next2;
+ const ip_adjacency_t * adj2;
+ const dpo_id_t *dpo2;
+ vlib_buffer_t * b2;
+ u32 bi3, adj_index3, next3;
+ const ip_adjacency_t * adj3;
+ const dpo_id_t *dpo3;
+ vlib_buffer_t * b3;
/* Prefetch next iteration. */
{
- vlib_buffer_t * p2, * p3;
-
- p2 = vlib_get_buffer (vm, from[2]);
- p3 = vlib_get_buffer (vm, from[3]);
-
- vlib_prefetch_buffer_header (p2, LOAD);
- vlib_prefetch_buffer_header (p3, LOAD);
-
- CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
- CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+ vlib_buffer_t * p4, * p5;
+ vlib_buffer_t * p6, * p7;
+
+ p4 = vlib_get_buffer (vm, from[4]);
+ p5 = vlib_get_buffer (vm, from[5]);
+ p6 = vlib_get_buffer (vm, from[6]);
+ p7 = vlib_get_buffer (vm, from[7]);
+
+ vlib_prefetch_buffer_header (p4, LOAD);
+ vlib_prefetch_buffer_header (p5, LOAD);
+ vlib_prefetch_buffer_header (p6, LOAD);
+ vlib_prefetch_buffer_header (p7, LOAD);
}
bi0 = from[0];
to_next[0] = bi0;
bi1 = from[1];
to_next[1] = bi1;
+ bi2 = from[2];
+ to_next[2] = bi2;
+ bi3 = from[3];
+ to_next[3] = bi3;
- from += 2;
- to_next += 2;
- n_left_from -= 2;
- n_left_to_next -= 2;
+ from += 4;
+ to_next += 4;
+ n_left_from -= 4;
+ n_left_to_next -= 4;
b0 = vlib_get_buffer(vm, bi0);
b1 = vlib_get_buffer(vm, bi1);
+ b2 = vlib_get_buffer(vm, bi2);
+ b3 = vlib_get_buffer(vm, bi3);
/* Follow the DPO on which the midchain is stacked */
adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
+ adj_index2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
+ adj_index3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
adj0 = adj_get(adj_index0);
adj1 = adj_get(adj_index1);
+ adj2 = adj_get(adj_index2);
+ adj3 = adj_get(adj_index3);
dpo0 = &adj0->sub_type.midchain.next_dpo;
dpo1 = &adj1->sub_type.midchain.next_dpo;
+ dpo2 = &adj2->sub_type.midchain.next_dpo;
+ dpo3 = &adj3->sub_type.midchain.next_dpo;
next0 = dpo0->dpoi_next_node;
next1 = dpo1->dpoi_next_node;
+ next2 = dpo2->dpoi_next_node;
+ next3 = dpo3->dpoi_next_node;
- vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
- vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
+ vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
+ vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
+ vnet_buffer(b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
+ vnet_buffer(b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
if (interface_count)
{
vlib_increment_combined_counter (im->combined_sw_if_counters
+ VNET_INTERFACE_COUNTER_TX,
- cpu_index,
+ thread_index,
adj0->rewrite_header.sw_if_index,
1,
vlib_buffer_length_in_chain (vm, b0));
vlib_increment_combined_counter (im->combined_sw_if_counters
+ VNET_INTERFACE_COUNTER_TX,
- cpu_index,
+ thread_index,
adj1->rewrite_header.sw_if_index,
1,
vlib_buffer_length_in_chain (vm, b1));
+ vlib_increment_combined_counter (im->combined_sw_if_counters
+ + VNET_INTERFACE_COUNTER_TX,
+ thread_index,
+ adj2->rewrite_header.sw_if_index,
+ 1,
+ vlib_buffer_length_in_chain (vm, b2));
+ vlib_increment_combined_counter (im->combined_sw_if_counters
+ + VNET_INTERFACE_COUNTER_TX,
+ thread_index,
+ adj3->rewrite_header.sw_if_index,
+ 1,
+ vlib_buffer_length_in_chain (vm, b3));
}
if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
b1, sizeof (*tr));
tr->ai = adj_index1;
}
+ if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
+ b2, sizeof (*tr));
+ tr->ai = adj_index2;
+ }
+ if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
+ b3, sizeof (*tr));
+ tr->ai = adj_index3;
+ }
- vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+ vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
to_next, n_left_to_next,
- bi0, bi1,
- next0, next1);
+ bi0, bi1, bi2, bi3,
+ next0, next1, next2, next3);
}
while (n_left_from > 0 && n_left_to_next > 0)
{
adj0 = adj_get(adj_index0);
dpo0 = &adj0->sub_type.midchain.next_dpo;
next0 = dpo0->dpoi_next_node;
- vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
+ vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
if (interface_count)
{
vlib_increment_combined_counter (im->combined_sw_if_counters
+ VNET_INTERFACE_COUNTER_TX,
- cpu_index,
+ thread_index,
adj0->rewrite_header.sw_if_index,
1,
vlib_buffer_length_in_chain (vm, b0));
static u8
adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
{
- u8 arc = (u8) ~0;
+ u8 arc = (u8) ~0;
switch (adj->ia_link)
{
case VNET_LINK_IP4:
return (arc);
}
+static u32
+adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
+{
+ return ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
+ adj_midchain_tx_no_count_node.index :
+ adj_midchain_tx_node.index);
+}
+
+static u32
+adj_nbr_midchain_get_feature_node (ip_adjacency_t *adj)
+{
+ if (adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT)
+ {
+ return (adj_midchain_tx_no_count_feature_node[adj->ia_link]);
+ }
+
+ return (adj_midchain_tx_feature_node[adj->ia_link]);
+}
+
+/**
+ * adj_midchain_setup
+ *
+ * Setup the adj as a mid-chain
+ */
+void
+adj_midchain_setup (adj_index_t adj_index,
+ adj_midchain_fixup_t fixup,
+ const void *data,
+ adj_flags_t flags)
+{
+ u32 feature_index, tx_node;
+ ip_adjacency_t *adj;
+ u8 arc_index;
+
+ ASSERT(ADJ_INDEX_INVALID != adj_index);
+
+ adj = adj_get(adj_index);
+
+ adj->sub_type.midchain.fixup_func = fixup;
+ adj->sub_type.midchain.fixup_data = data;
+ adj->ia_flags |= flags;
+
+ arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
+ feature_index = adj_nbr_midchain_get_feature_node(adj);
+ tx_node = adj_nbr_midchain_get_tx_node(adj);
+
+ vnet_feature_enable_disable_with_index (arc_index, feature_index,
+ adj->rewrite_header.sw_if_index,
+ 1 /* enable */, 0, 0);
+
+ /*
+ * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
+ * The graph arc used/created here is from the midchain-tx node to the
+ * child's registered node. This is because post adj processing the next
+ * node are any output features, then the midchain-tx. from there we
+ * need to get to the stacked child's node.
+ */
+ dpo_stack_from_node(tx_node,
+ &adj->sub_type.midchain.next_dpo,
+ drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
+}
+
/**
* adj_nbr_midchain_update_rewrite
*
void
adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
adj_midchain_fixup_t fixup,
- adj_midchain_flag_t flags,
+ const void *fixup_data,
+ adj_flags_t flags,
u8 *rewrite)
{
ip_adjacency_t *adj;
- u8 arc_index;
- u32 feature_index;
ASSERT(ADJ_INDEX_INVALID != adj_index);
*/
ASSERT(NULL != rewrite);
- adj->sub_type.midchain.fixup_func = fixup;
-
- arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
- feature_index = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
- adj_midchain_tx_no_count_feature_node[adj->ia_link] :
- adj_midchain_tx_feature_node[adj->ia_link];
-
- adj->sub_type.midchain.tx_function_node = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
- adj_midchain_tx_no_count_node.index :
- adj_midchain_tx_node.index;
-
- vnet_feature_enable_disable_with_index (arc_index, feature_index,
- adj->rewrite_header.sw_if_index,
- 1 /* enable */, 0, 0);
-
- /*
- * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
- * The graph arc used/created here is from the midchain-tx node to the
- * child's registered node. This is because post adj processing the next
- * node are any output features, then the midchain-tx. from there we
- * need to get to the stacked child's node.
- */
- dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
- &adj->sub_type.midchain.next_dpo,
- drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
+ adj_midchain_setup(adj_index, fixup, fixup_data, flags);
/*
* update the rewirte with the workers paused.
adj_nbr_update_rewrite_internal(adj,
IP_LOOKUP_NEXT_MIDCHAIN,
adj_get_midchain_node(adj->ia_link),
- adj->sub_type.midchain.tx_function_node,
+ adj_nbr_midchain_get_tx_node(adj),
rewrite);
}
* stack on the drop
*/
dpo_stack(DPO_ADJACENCY_MIDCHAIN,
- vnet_link_to_dpo_proto(adj->ia_link),
- &adj->sub_type.midchain.next_dpo,
- drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
-
+ vnet_link_to_dpo_proto(adj->ia_link),
+ &adj->sub_type.midchain.next_dpo,
+ drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
CLIB_MEMORY_BARRIER();
}
adj = adj_get(adj_index);
- ASSERT(IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index);
+ ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) ||
+ (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index));
- dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
+ dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
&adj->sub_type.midchain.next_dpo,
next);
}
format_vnet_rewrite,
&adj->rewrite_header, sizeof (adj->rewrite_data), indent);
s = format (s, "\n%Ustacked-on:\n%U%U",
- format_white_space, indent,
- format_white_space, indent+2,
- format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
+ format_white_space, indent,
+ format_white_space, indent+2,
+ format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
return (s);
}
.dv_lock = adj_dpo_lock,
.dv_unlock = adj_dpo_unlock,
.dv_format = format_adj_midchain,
+ .dv_get_urpf = adj_dpo_get_urpf,
};
/**
Code Review / vpp.git / blobdiff