- struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
-
- /* prefetches are interleaved with the rest of the code to reduce
- pressure on L1 cache */
- dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 8]);
- dpdk_prefetch_ethertype (xd->rx_vectors[queue_id][mb_index + 4]);
-
- mb0 = xd->rx_vectors[queue_id][mb_index];
- mb1 = xd->rx_vectors[queue_id][mb_index + 1];
- mb2 = xd->rx_vectors[queue_id][mb_index + 2];
- mb3 = xd->rx_vectors[queue_id][mb_index + 3];
-
- ASSERT (mb0);
- ASSERT (mb1);
- ASSERT (mb2);
- ASSERT (mb3);
-
- if (maybe_multiseg)
- {
- if (PREDICT_FALSE (mb0->nb_segs > 1))
- dpdk_prefetch_buffer (mb0->next);
- if (PREDICT_FALSE (mb1->nb_segs > 1))
- dpdk_prefetch_buffer (mb1->next);
- if (PREDICT_FALSE (mb2->nb_segs > 1))
- dpdk_prefetch_buffer (mb2->next);
- if (PREDICT_FALSE (mb3->nb_segs > 1))
- dpdk_prefetch_buffer (mb3->next);
- }
-
- b0 = vlib_buffer_from_rte_mbuf (mb0);
- b1 = vlib_buffer_from_rte_mbuf (mb1);
- b2 = vlib_buffer_from_rte_mbuf (mb2);
- b3 = vlib_buffer_from_rte_mbuf (mb3);
-
- dpdk_buffer_init_from_template (b0, b1, b2, b3, bt);
-
- dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 9]);
- dpdk_prefetch_ethertype (xd->rx_vectors[queue_id][mb_index + 5]);
-
- /* current_data must be set to -RTE_PKTMBUF_HEADROOM in template */
- b0->current_data += mb0->data_off;
- b1->current_data += mb1->data_off;
- b2->current_data += mb2->data_off;
- b3->current_data += mb3->data_off;
-
- b0->current_length = mb0->data_len;
- b1->current_length = mb1->data_len;
- b2->current_length = mb2->data_len;
- b3->current_length = mb3->data_len;
-
- dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 10]);
- dpdk_prefetch_ethertype (xd->rx_vectors[queue_id][mb_index + 7]);
-
- bi0 = vlib_get_buffer_index (vm, b0);
- bi1 = vlib_get_buffer_index (vm, b1);
- bi2 = vlib_get_buffer_index (vm, b2);
- bi3 = vlib_get_buffer_index (vm, b3);
-
- to_next[0] = bi0;
- to_next[1] = bi1;
- to_next[2] = bi2;
- to_next[3] = bi3;
- to_next += 4;
- n_left_to_next -= 4;
-
- if (PREDICT_FALSE (xd->per_interface_next_index != ~0))
- {
- next0 = next1 = next2 = next3 = xd->per_interface_next_index;
- }
- else
- {
- next0 = dpdk_rx_next_from_etype (mb0, b0);
- next1 = dpdk_rx_next_from_etype (mb1, b1);
- next2 = dpdk_rx_next_from_etype (mb2, b2);
- next3 = dpdk_rx_next_from_etype (mb3, b3);
- }
-
- dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 11]);
- dpdk_prefetch_ethertype (xd->rx_vectors[queue_id][mb_index + 6]);
-
- or_ol_flags = (mb0->ol_flags | mb1->ol_flags |
- mb2->ol_flags | mb3->ol_flags);
- if (PREDICT_FALSE (or_ol_flags & PKT_RX_IP_CKSUM_BAD))
- {
- dpdk_rx_error_from_mb (mb0, &next0, &error0);
- dpdk_rx_error_from_mb (mb1, &next1, &error1);
- dpdk_rx_error_from_mb (mb2, &next2, &error2);
- dpdk_rx_error_from_mb (mb3, &next3, &error3);
- b0->error = node->errors[error0];
- b1->error = node->errors[error1];
- b2->error = node->errors[error2];
- b3->error = node->errors[error3];
- }
-
- vlib_buffer_advance (b0, device_input_next_node_advance[next0]);
- vlib_buffer_advance (b1, device_input_next_node_advance[next1]);
- vlib_buffer_advance (b2, device_input_next_node_advance[next2]);
- vlib_buffer_advance (b3, device_input_next_node_advance[next3]);
-
- n_rx_bytes += mb0->pkt_len;
- n_rx_bytes += mb1->pkt_len;
- n_rx_bytes += mb2->pkt_len;
- n_rx_bytes += mb3->pkt_len;
-
- /* Process subsequent segments of multi-segment packets */
- if (maybe_multiseg)
- {
- dpdk_process_subseq_segs (vm, b0, mb0, fl);
- dpdk_process_subseq_segs (vm, b1, mb1, fl);
- dpdk_process_subseq_segs (vm, b2, mb2, fl);
- dpdk_process_subseq_segs (vm, b3, mb3, fl);
- }
-
- /*
- * Turn this on if you run into
- * "bad monkey" contexts, and you want to know exactly
- * which nodes they've visited... See main.c...
- */
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3);
-
- /* Do we have any driver RX features configured on the interface? */
- vnet_feature_start_device_input_x4 (xd->vlib_sw_if_index,
- &next0, &next1, &next2, &next3,
- b0, b1, b2, b3);
-
- vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
- to_next, n_left_to_next,
- bi0, bi1, bi2, bi3,
- next0, next1, next2, next3);
- n_buffers -= 4;
- mb_index += 4;