- while (n_left_from >= 8 && n_left_to_next > 4)
- {
- u32 bi0, adj_index0, next0;
- const ip_adjacency_t * adj0;
- const dpo_id_t *dpo0;
- vlib_buffer_t * b0;
- u32 bi1, adj_index1, next1;
- const ip_adjacency_t * adj1;
- const dpo_id_t *dpo1;
- vlib_buffer_t * b1;
- u32 bi2, adj_index2, next2;
- const ip_adjacency_t * adj2;
- const dpo_id_t *dpo2;
- vlib_buffer_t * b2;
- u32 bi3, adj_index3, next3;
- const ip_adjacency_t * adj3;
- const dpo_id_t *dpo3;
- vlib_buffer_t * b3;
-
- /* Prefetch next iteration. */
- {
- vlib_buffer_t * p4, * p5;
- vlib_buffer_t * p6, * p7;
-
- p4 = vlib_get_buffer (vm, from[4]);
- p5 = vlib_get_buffer (vm, from[5]);
- p6 = vlib_get_buffer (vm, from[6]);
- p7 = vlib_get_buffer (vm, from[7]);
-
- vlib_prefetch_buffer_header (p4, LOAD);
- vlib_prefetch_buffer_header (p5, LOAD);
- vlib_prefetch_buffer_header (p6, LOAD);
- vlib_prefetch_buffer_header (p7, LOAD);
- }
-
- bi0 = from[0];
- to_next[0] = bi0;
- bi1 = from[1];
- to_next[1] = bi1;
- bi2 = from[2];
- to_next[2] = bi2;
- bi3 = from[3];
- to_next[3] = bi3;
-
- from += 4;
- to_next += 4;
- n_left_from -= 4;
- n_left_to_next -= 4;
-
- b0 = vlib_get_buffer(vm, bi0);
- b1 = vlib_get_buffer(vm, bi1);
- b2 = vlib_get_buffer(vm, bi2);
- b3 = vlib_get_buffer(vm, bi3);
-
- /* Follow the DPO on which the midchain is stacked */
- adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
- adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
- adj_index2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
- adj_index3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
-
- adj0 = adj_get(adj_index0);
- adj1 = adj_get(adj_index1);
- adj2 = adj_get(adj_index2);
- adj3 = adj_get(adj_index3);
-
- dpo0 = &adj0->sub_type.midchain.next_dpo;
- dpo1 = &adj1->sub_type.midchain.next_dpo;
- dpo2 = &adj2->sub_type.midchain.next_dpo;
- dpo3 = &adj3->sub_type.midchain.next_dpo;
-
- next0 = dpo0->dpoi_next_node;
- next1 = dpo1->dpoi_next_node;
- next2 = dpo2->dpoi_next_node;
- next3 = dpo3->dpoi_next_node;
-
- vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
- vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
- vnet_buffer(b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
- vnet_buffer(b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
-
- if (interface_count)
- {
- vlib_increment_combined_counter (im->combined_sw_if_counters
- + VNET_INTERFACE_COUNTER_TX,
- thread_index,
- adj0->rewrite_header.sw_if_index,
- 1,
- vlib_buffer_length_in_chain (vm, b0));
- vlib_increment_combined_counter (im->combined_sw_if_counters
- + VNET_INTERFACE_COUNTER_TX,
- thread_index,
- adj1->rewrite_header.sw_if_index,
- 1,
- vlib_buffer_length_in_chain (vm, b1));
- vlib_increment_combined_counter (im->combined_sw_if_counters
- + VNET_INTERFACE_COUNTER_TX,
- thread_index,
- adj2->rewrite_header.sw_if_index,
- 1,
- vlib_buffer_length_in_chain (vm, b2));
- vlib_increment_combined_counter (im->combined_sw_if_counters
- + VNET_INTERFACE_COUNTER_TX,
- thread_index,
- adj3->rewrite_header.sw_if_index,
- 1,
- vlib_buffer_length_in_chain (vm, b3));
- }
-
- if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
- {
- adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
- b0, sizeof (*tr));
- tr->ai = adj_index0;
- }
- if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
- {
- adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
- b1, sizeof (*tr));
- tr->ai = adj_index1;
- }
- if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
- {
- adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
- b2, sizeof (*tr));
- tr->ai = adj_index2;
- }
- if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
- {
- adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
- b3, sizeof (*tr));
- tr->ai = adj_index3;
- }
-
- vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
- to_next, n_left_to_next,
- bi0, bi1, bi2, bi3,
- next0, next1, next2, next3);
- }
- while (n_left_from > 0 && n_left_to_next > 0)
- {
- u32 bi0, adj_index0, next0;
- const ip_adjacency_t * adj0;
- const dpo_id_t *dpo0;
- vlib_buffer_t * b0;
-
- bi0 = from[0];
- to_next[0] = bi0;
- from += 1;
- to_next += 1;
- n_left_from -= 1;
- n_left_to_next -= 1;
-
- b0 = vlib_get_buffer(vm, bi0);
-
- /* Follow the DPO on which the midchain is stacked */
- adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
- adj0 = adj_get(adj_index0);
- dpo0 = &adj0->sub_type.midchain.next_dpo;
- next0 = dpo0->dpoi_next_node;
- vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
-
- if (interface_count)
- {
- vlib_increment_combined_counter (im->combined_sw_if_counters
- + VNET_INTERFACE_COUNTER_TX,
- thread_index,
- adj0->rewrite_header.sw_if_index,
- 1,
- vlib_buffer_length_in_chain (vm, b0));
- }
-
- if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
- {
- adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
- b0, sizeof (*tr));
- tr->ai = adj_index0;
- }
-
- vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
- to_next, n_left_to_next,
- bi0, next0);
- }
+ /* Prefetch next iteration. */
+ {
+ vlib_prefetch_buffer_header (b[4], LOAD);
+ vlib_prefetch_buffer_header (b[5], LOAD);
+ vlib_prefetch_buffer_header (b[6], LOAD);
+ vlib_prefetch_buffer_header (b[7], LOAD);
+ }
+
+ /* Follow the DPO on which the midchain is stacked */
+ adj_index0 = vnet_buffer(b[0])->ip.adj_index[VLIB_TX];
+ adj_index1 = vnet_buffer(b[1])->ip.adj_index[VLIB_TX];
+ adj_index2 = vnet_buffer(b[2])->ip.adj_index[VLIB_TX];
+ adj_index3 = vnet_buffer(b[3])->ip.adj_index[VLIB_TX];
+
+ adj0 = adj_get(adj_index0);
+ adj1 = adj_get(adj_index1);
+ adj2 = adj_get(adj_index2);
+ adj3 = adj_get(adj_index3);
+
+ dpo0 = &adj0->sub_type.midchain.next_dpo;
+ dpo1 = &adj1->sub_type.midchain.next_dpo;
+ dpo2 = &adj2->sub_type.midchain.next_dpo;
+ dpo3 = &adj3->sub_type.midchain.next_dpo;
+
+ next[0] = dpo0->dpoi_next_node;
+ next[1] = dpo1->dpoi_next_node;
+ next[2] = dpo2->dpoi_next_node;
+ next[3] = dpo3->dpoi_next_node;
+
+ vnet_buffer(b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
+ vnet_buffer(b[1])->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
+ vnet_buffer(b[2])->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
+ vnet_buffer(b[3])->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
+
+ if (interface_count)
+ {
+ vlib_increment_combined_counter (im->combined_sw_if_counters
+ + VNET_INTERFACE_COUNTER_TX,
+ thread_index,
+ adj0->rewrite_header.sw_if_index,
+ 1,
+ vlib_buffer_length_in_chain (vm, b[0]));
+ vlib_increment_combined_counter (im->combined_sw_if_counters
+ + VNET_INTERFACE_COUNTER_TX,
+ thread_index,
+ adj1->rewrite_header.sw_if_index,
+ 1,
+ vlib_buffer_length_in_chain (vm, b[1]));
+ vlib_increment_combined_counter (im->combined_sw_if_counters
+ + VNET_INTERFACE_COUNTER_TX,
+ thread_index,
+ adj2->rewrite_header.sw_if_index,
+ 1,
+ vlib_buffer_length_in_chain (vm, b[2]));
+ vlib_increment_combined_counter (im->combined_sw_if_counters
+ + VNET_INTERFACE_COUNTER_TX,
+ thread_index,
+ adj3->rewrite_header.sw_if_index,
+ 1,
+ vlib_buffer_length_in_chain (vm, b[3]));
+ }
+
+ if (PREDICT_FALSE(node->flags & VLIB_NODE_FLAG_TRACE))
+ {
+ if (PREDICT_FALSE(b[0]->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
+ b[0], sizeof (*tr));
+ tr->ai = adj_index0;
+ }
+ if (PREDICT_FALSE(b[1]->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
+ b[1], sizeof (*tr));
+ tr->ai = adj_index1;
+ }
+ if (PREDICT_FALSE(b[2]->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
+ b[2], sizeof (*tr));
+ tr->ai = adj_index2;
+ }
+ if (PREDICT_FALSE(b[3]->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
+ b[3], sizeof (*tr));
+ tr->ai = adj_index3;
+ }
+ }
+ n_left -= 4;
+ b += 4;
+ next += 4;
+ }
+
+ while (n_left)
+ {
+ const ip_adjacency_t * adj0;
+ const dpo_id_t *dpo0;
+ u32 adj_index0;
+
+ /* Follow the DPO on which the midchain is stacked */
+ adj_index0 = vnet_buffer(b[0])->ip.adj_index[VLIB_TX];
+ adj0 = adj_get(adj_index0);
+ dpo0 = &adj0->sub_type.midchain.next_dpo;
+ next[0] = dpo0->dpoi_next_node;
+ vnet_buffer(b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
+
+ if (interface_count)
+ {
+ vlib_increment_combined_counter (im->combined_sw_if_counters
+ + VNET_INTERFACE_COUNTER_TX,
+ thread_index,
+ adj0->rewrite_header.sw_if_index,
+ 1,
+ vlib_buffer_length_in_chain (vm, b[0]));
+ }