X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fdevice%2Fnode.c;h=25222856912ddf0a78de5d5d5a341eb02af35d22;hb=refs%2Fchanges%2F46%2F33146%2F2;hp=7db66ffab256e66d4e24714e73442ad36804aa56;hpb=a5c308e68d5578e5d75327619e30d73a95abb1a5;p=vpp.git diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index 7db66ffab25..25222856912 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -85,10 +86,10 @@ dpdk_process_subseq_segs (vlib_main_t * vm, vlib_buffer_t * b, static_always_inline void dpdk_prefetch_mbuf_x4 (struct rte_mbuf *mb[]) { - CLIB_PREFETCH (mb[0], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (mb[1], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (mb[2], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (mb[3], CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (mb[0]); + clib_prefetch_load (mb[1]); + clib_prefetch_load (mb[2]); + clib_prefetch_load (mb[3]); } static_always_inline void @@ -96,13 +97,13 @@ dpdk_prefetch_buffer_x4 (struct rte_mbuf *mb[]) { vlib_buffer_t *b; b = vlib_buffer_from_rte_mbuf (mb[0]); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); b = vlib_buffer_from_rte_mbuf (mb[1]); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); b = vlib_buffer_from_rte_mbuf (mb[2]); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); b = vlib_buffer_from_rte_mbuf (mb[3]); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); } /** \brief Main DPDK input node @@ -144,16 +145,16 @@ dpdk_prefetch_buffer_x4 (struct rte_mbuf *mb[]) xd->per_interface_next_index */ -static_always_inline u8 -dpdk_ol_flags_extract (struct rte_mbuf **mb, u8 * flags, int count) +static_always_inline u16 +dpdk_ol_flags_extract (struct rte_mbuf **mb, u16 * flags, int count) { - u8 rv = 0; + u16 rv = 0; int i; for (i = 0; i < count; i++) { /* all flags we are interested in are in lower 8 bits but that might change */ - flags[i] = (u8) mb[i]->ol_flags; + flags[i] = (u16) mb[i]->ol_flags; rv |= flags[i]; } return rv; @@ -161,13 +162,14 @@ dpdk_ol_flags_extract (struct rte_mbuf **mb, u8 * flags, int count) static_always_inline uword dpdk_process_rx_burst (vlib_main_t * vm, dpdk_per_thread_data_t * ptd, - uword n_rx_packets, int maybe_multiseg, u8 * or_flagsp) + uword n_rx_packets, int maybe_multiseg, + u16 * or_flagsp) { u32 n_left = n_rx_packets; vlib_buffer_t *b[4]; struct rte_mbuf **mb = ptd->mbufs; uword n_bytes = 0; - u8 *flags, or_flags = 0; + u16 *flags, or_flags = 0; vlib_buffer_t bt; mb = ptd->mbufs; @@ -214,11 +216,6 @@ dpdk_process_rx_burst (vlib_main_t * vm, dpdk_per_thread_data_t * ptd, n_bytes += dpdk_process_subseq_segs (vm, b[3], mb[3], &bt); } - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]); - /* next */ mb += 4; n_left -= 4; @@ -236,7 +233,6 @@ dpdk_process_rx_burst (vlib_main_t * vm, dpdk_per_thread_data_t * ptd, if (maybe_multiseg) n_bytes += dpdk_process_subseq_segs (vm, b[0], mb[0], &bt); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); /* next */ mb += 1; @@ -258,7 +254,7 @@ dpdk_process_flow_offload (dpdk_device_t * xd, dpdk_per_thread_data_t * ptd, /* TODO prefetch and quad-loop */ for (n = 0; n < n_rx_packets; n++) { - if ((ptd->flags[n] & (1 << PKT_RX_FDIR)) == 0) + if ((ptd->flags[n] & PKT_RX_FDIR_ID) == 0) continue; fle = pool_elt_at_index (xd->flow_lookup_entries, @@ -286,13 +282,14 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, vlib_node_runtime_t * node, u32 thread_index, u16 queue_id) { uword n_rx_packets = 0, n_rx_bytes; + dpdk_rx_queue_t *rxq = vec_elt_at_index (xd->rx_queues, queue_id); u32 n_left, n_trace; u32 *buffers; u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; struct rte_mbuf **mb; vlib_buffer_t *b0; u16 *next; - u8 or_flags; + u16 or_flags; u32 n; int single_next = 0; @@ -323,7 +320,8 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, bt->error = node->errors[DPDK_ERROR_NONE]; /* as DPDK is allocating empty buffers from mempool provided before interface start for each queue, it is safe to store this in the template */ - bt->buffer_pool_index = xd->buffer_pool_for_queue[queue_id]; + bt->buffer_pool_index = rxq->buffer_pool_index; + bt->ref_count = 1; vnet_buffer (bt)->feature_arc_index = 0; bt->current_config_index = 0; @@ -376,7 +374,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, vlib_frame_t *f; ethernet_input_frame_t *ef; nf = vlib_node_runtime_get_next_frame (vm, node, next_index); - f = vlib_get_frame (vm, nf->frame_index); + f = vlib_get_frame (vm, nf->frame); f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; ef = vlib_frame_scalar_args (f); @@ -389,6 +387,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, if (xd->flags & DPDK_DEVICE_FLAG_RX_IP4_CKSUM && (or_flags & PKT_RX_IP_CKSUM_BAD) == 0) f->flags |= ETH_INPUT_FRAME_F_IP4_CKSUM_OK; + vlib_frame_no_append (f); } n_left_to_next -= n_rx_packets; vlib_put_next_frame (vm, node, next_index, n_left_to_next); @@ -413,21 +412,28 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, b0 = vlib_get_buffer (vm, buffers[0]); if (single_next == 0) next_index = next[0]; - vlib_trace_buffer (vm, node, next_index, b0, /* follow_chain */ 0); - - dpdk_rx_trace_t *t0 = vlib_add_trace (vm, node, b0, sizeof t0[0]); - t0->queue_index = queue_id; - t0->device_index = xd->device_index; - t0->buffer_index = vlib_get_buffer_index (vm, b0); - - clib_memcpy_fast (&t0->mb, mb[0], sizeof t0->mb); - clib_memcpy_fast (&t0->buffer, b0, - sizeof b0[0] - sizeof b0->pre_data); - clib_memcpy_fast (t0->buffer.pre_data, b0->data, - sizeof t0->buffer.pre_data); - clib_memcpy_fast (&t0->data, mb[0]->buf_addr + mb[0]->data_off, - sizeof t0->data); - n_trace--; + + if (PREDICT_TRUE + (vlib_trace_buffer + (vm, node, next_index, b0, /* follow_chain */ 0))) + { + + dpdk_rx_trace_t *t0 = + vlib_add_trace (vm, node, b0, sizeof t0[0]); + t0->queue_index = queue_id; + t0->device_index = xd->device_index; + t0->buffer_index = vlib_get_buffer_index (vm, b0); + + clib_memcpy_fast (&t0->mb, mb[0], sizeof t0->mb); + clib_memcpy_fast (&t0->buffer, b0, + sizeof b0[0] - sizeof b0->pre_data); + clib_memcpy_fast (t0->buffer.pre_data, b0->data, + sizeof t0->buffer.pre_data); + clib_memcpy_fast (&t0->data, mb[0]->buf_addr + mb[0]->data_off, + sizeof t0->data); + n_trace--; + } + n_left--; buffers++; mb++; @@ -436,48 +442,6 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, vlib_set_trace_count (vm, node, n_trace); } - /* rx pcap capture if enabled */ - if (PREDICT_FALSE (dm->pcap[VLIB_RX].pcap_enable)) - { - u32 bi0; - n_left = n_rx_packets; - buffers = ptd->buffers; - while (n_left) - { - bi0 = buffers[0]; - b0 = vlib_get_buffer (vm, bi0); - buffers++; - - if (dm->pcap[VLIB_RX].pcap_sw_if_index == 0 || - dm->pcap[VLIB_RX].pcap_sw_if_index - == vnet_buffer (b0)->sw_if_index[VLIB_RX]) - { - struct rte_mbuf *mb; - i16 data_start; - i32 temp_advance; - - /* - * Note: current_data will have advanced - * when we skip ethernet input. - * Temporarily back up to the original DMA - * target, so we capture a valid ethernet frame - */ - mb = rte_mbuf_from_vlib_buffer (b0); - - /* Figure out the original data_start */ - data_start = (mb->buf_addr + mb->data_off) - (void *) b0->data; - /* Back up that far */ - temp_advance = b0->current_data - data_start; - vlib_buffer_advance (b0, -temp_advance); - /* Trace the packet */ - pcap_add_buffer (&dm->pcap[VLIB_RX].pcap_main, vm, bi0, 512); - /* and advance again */ - vlib_buffer_advance (b0, temp_advance); - } - n_left--; - } - } - vlib_increment_combined_counter (vnet_get_main ()->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, xd->sw_if_index, @@ -494,23 +458,21 @@ VLIB_NODE_FN (dpdk_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node, dpdk_main_t *dm = &dpdk_main; dpdk_device_t *xd; uword n_rx_packets = 0; - vnet_device_input_runtime_t *rt = (void *) node->runtime_data; - vnet_device_and_queue_t *dq; + vnet_hw_if_rxq_poll_vector_t *pv; u32 thread_index = node->thread_index; /* * Poll all devices on this cpu for input/interrupts. */ - /* *INDENT-OFF* */ - foreach_device_and_queue (dq, rt->devices_and_queues) + + pv = vnet_hw_if_get_rxq_poll_vector (vm, node); + + for (int i = 0; i < vec_len (pv); i++) { - xd = vec_elt_at_index(dm->devices, dq->dev_instance); - if (PREDICT_FALSE (xd->flags & DPDK_DEVICE_FLAG_BOND_SLAVE)) - continue; /* Do not poll slave to a bonded interface */ - n_rx_packets += dpdk_device_input (vm, dm, xd, node, thread_index, - dq->queue_id); + xd = vec_elt_at_index (dm->devices, pv[i].dev_instance); + n_rx_packets += + dpdk_device_input (vm, dm, xd, node, thread_index, pv[i].queue_id); } - /* *INDENT-ON* */ return n_rx_packets; } @@ -519,6 +481,7 @@ VLIB_REGISTER_NODE (dpdk_input_node) = { .type = VLIB_NODE_TYPE_INPUT, .name = "dpdk-input", .sibling_of = "device-input", + .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, /* Will be enabled if/when hardware is detected. */ .state = VLIB_NODE_STATE_DISABLED,