X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fdevice%2Fnode.c;h=602891dbabe0907f38985632cbb2ef6da12f0da6;hb=af86a48;hp=3311ac4830dfeacd6caea1bd077333c4c55c08c4;hpb=afe56de947822bb981bd30242f4e3c2c469f9ecc;p=vpp.git diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index 3311ac4830d..602891dbabe 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -28,7 +28,7 @@ #include -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static char *dpdk_error_strings[] = { #define _(n,s) s, foreach_dpdk_error @@ -474,7 +474,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, /* get up to DPDK_RX_BURST_SZ buffers from PMD */ while (n_rx_packets < DPDK_RX_BURST_SZ) { - n = rte_eth_rx_burst (xd->device_index, queue_id, + n = rte_eth_rx_burst (xd->port_id, queue_id, ptd->mbufs + n_rx_packets, DPDK_RX_BURST_SZ - n_rx_packets); n_rx_packets += n; @@ -500,7 +500,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, next_index = xd->per_interface_next_index; } - /* as all packets belong to thr same interface feature arc lookup + /* as all packets belong to the same interface feature arc lookup can be don once and result stored in the buffer template */ if (PREDICT_FALSE (vnet_device_input_have_features (xd->sw_if_index))) { @@ -548,84 +548,12 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, vlib_get_buffer_indices_with_offset (vm, (void **) ptd->mbufs, ptd->buffers, n_rx_packets, sizeof (struct rte_mbuf)); - n_left = n_rx_packets; - next = ptd->next; - buffers = ptd->buffers; - mb = ptd->mbufs; - while (n_left) - { - u32 n_left_to_next; - u32 *to_next; - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); -#ifdef CLIB_HAVE_VEC256 - while (n_left >= 16 && n_left_to_next >= 16) - { - u16x16 next16 = u16x16_load_unaligned (next); - if (u16x16_is_all_equal (next16, next_index)) - { - clib_memcpy (to_next, buffers, 16 * sizeof (u32)); - to_next += 16; - n_left_to_next -= 16; - buffers += 16; - n_left -= 16; - next += 16; - mb += 16; - } - else - { - clib_memcpy (to_next, buffers, 4 * sizeof (u32)); - to_next += 4; - n_left_to_next -= 4; - - vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, - n_left_to_next, buffers[0], - buffers[1], buffers[2], - buffers[3], next[0], next[1], - next[2], next[3]); - /* next */ - buffers += 4; - n_left -= 4; - next += 4; - mb += 4; - } - } -#endif - while (n_left >= 4 && n_left_to_next >= 4) - { - clib_memcpy (to_next, buffers, 4 * sizeof (u32)); - to_next += 4; - n_left_to_next -= 4; - - vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, - n_left_to_next, buffers[0], - buffers[1], buffers[2], buffers[3], - next[0], next[1], next[2], - next[3]); - /* next */ - buffers += 4; - n_left -= 4; - next += 4; - mb += 4; - } - while (n_left && n_left_to_next) - { - clib_memcpy (to_next, buffers, 1 * sizeof (u32)); - to_next += 1; - n_left_to_next -= 1; - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, - n_left_to_next, buffers[0], - next[0]); - /* next */ - buffers += 1; - n_left -= 1; - next += 1; - mb += 1; - } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } + + vlib_buffer_enqueue_to_next (vm, node, ptd->buffers, ptd->next, + n_rx_packets); /* packet trace if enabled */ - if ((n_trace = vlib_get_trace_count (vm, node))) + if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node)))) { n_left = n_rx_packets; buffers = ptd->buffers; @@ -656,6 +584,48 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, vlib_set_trace_count (vm, node, n_trace); } + /* rx pcap capture if enabled */ + if (PREDICT_FALSE (dm->pcap[VLIB_RX].pcap_enable)) + { + u32 bi0; + n_left = n_rx_packets; + buffers = ptd->buffers; + while (n_left) + { + bi0 = buffers[0]; + b0 = vlib_get_buffer (vm, bi0); + buffers++; + + if (dm->pcap[VLIB_RX].pcap_sw_if_index == 0 || + dm->pcap[VLIB_RX].pcap_sw_if_index + == vnet_buffer (b0)->sw_if_index[VLIB_RX]) + { + struct rte_mbuf *mb; + i16 data_start; + i32 temp_advance; + + /* + * Note: current_data will have advanced + * when we skip ethernet input. + * Temporarily back up to the original DMA + * target, so we capture a valid ethernet frame + */ + mb = rte_mbuf_from_vlib_buffer (b0); + + /* Figure out the original data_start */ + data_start = (mb->buf_addr + mb->data_off) - (void *) b0->data; + /* Back up that far */ + temp_advance = b0->current_data - data_start; + vlib_buffer_advance (b0, -temp_advance); + /* Trace the packet */ + pcap_add_buffer (&dm->pcap[VLIB_RX].pcap_main, vm, bi0, 512); + /* and advance again */ + vlib_buffer_advance (b0, temp_advance); + } + n_left--; + } + } + vlib_increment_combined_counter (vnet_get_main ()->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, xd->sw_if_index, @@ -666,8 +636,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, return n_rx_packets; } -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node, +VLIB_NODE_FN (dpdk_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f) { dpdk_main_t *dm = &dpdk_main; @@ -696,10 +665,9 @@ CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node, return n_rx_packets; } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT /* *INDENT-OFF* */ VLIB_REGISTER_NODE (dpdk_input_node) = { - .function = dpdk_input, .type = VLIB_NODE_TYPE_INPUT, .name = "dpdk-input", .sibling_of = "device-input", @@ -714,20 +682,6 @@ VLIB_REGISTER_NODE (dpdk_input_node) = { .error_strings = dpdk_error_strings, }; /* *INDENT-ON* */ - -vlib_node_function_t __clib_weak dpdk_input_avx512; -vlib_node_function_t __clib_weak dpdk_input_avx2; - -#if __x86_64__ -static void __clib_constructor -dpdk_input_multiarch_select (void) -{ - if (dpdk_input_avx512 && clib_cpu_supports_avx512f ()) - dpdk_input_node.function = dpdk_input_avx512; - else if (dpdk_input_avx2 && clib_cpu_supports_avx2 ()) - dpdk_input_node.function = dpdk_input_avx2; -} -#endif #endif /*