X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fdevice%2Fdevice.c;h=fcabe6e2dd6d1b28b4877c0e53b94fff1eb580eb;hb=3b81a1e5f205482b8ea30edbfd39559c4368ac4d;hp=cdd9785a3f375df25d461c4030e9a3e425a71b49;hpb=a35cc14d37466f0737fa928d25697fbfe6e7d657;p=vpp.git diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index cdd9785a3f3..fcabe6e2dd6 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -26,8 +26,7 @@ #define foreach_dpdk_tx_func_error \ _(BAD_RETVAL, "DPDK tx function returned an error") \ - _(PKT_DROP, "Tx packet drops (dpdk tx failure)") \ - _(REPL_FAIL, "Tx packet drops (replication failure)") + _(PKT_DROP, "Tx packet drops (dpdk tx failure)") typedef enum { @@ -37,7 +36,6 @@ typedef enum DPDK_TX_FUNC_N_ERROR, } dpdk_tx_func_error_t; -#ifndef CLIB_MULTIARCH_VARIANT static char *dpdk_tx_func_error_strings[] = { #define _(n,s) s, foreach_dpdk_tx_func_error @@ -45,13 +43,14 @@ static char *dpdk_tx_func_error_strings[] = { }; static clib_error_t * -dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address) +dpdk_set_mac_address (vnet_hw_interface_t * hi, + const u8 * old_address, const u8 * address) { int error; dpdk_main_t *dm = &dpdk_main; dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance); - error = rte_eth_dev_default_mac_addr_set (xd->device_index, + error = rte_eth_dev_default_mac_addr_set (xd->port_id, (struct ether_addr *) address); if (error) @@ -65,49 +64,6 @@ dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address) return NULL; } } -#endif - -static struct rte_mbuf * -dpdk_replicate_packet_mb (vlib_buffer_t * b) -{ - dpdk_main_t *dm = &dpdk_main; - struct rte_mbuf **mbufs = 0, *s, *d; - u8 nb_segs; - unsigned socket_id = rte_socket_id (); - int i; - - ASSERT (dm->pktmbuf_pools[socket_id]); - s = rte_mbuf_from_vlib_buffer (b); - nb_segs = s->nb_segs; - vec_validate (mbufs, nb_segs - 1); - - if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs)) - { - vec_free (mbufs); - return 0; - } - - d = mbufs[0]; - d->nb_segs = s->nb_segs; - d->data_len = s->data_len; - d->pkt_len = s->pkt_len; - d->data_off = s->data_off; - clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len); - - for (i = 1; i < nb_segs; i++) - { - d->next = mbufs[i]; - d = mbufs[i]; - s = s->next; - d->data_len = s->data_len; - clib_memcpy (d->buf_addr, s->buf_addr, - RTE_PKTMBUF_HEADROOM + s->data_len); - } - - d = mbufs[0]; - vec_free (mbufs); - return d; -} static void dpdk_tx_trace_buffer (dpdk_main_t * dm, vlib_node_runtime_t * node, @@ -226,7 +182,7 @@ static_always_inline else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD)) { /* no wrap, transmit in one burst */ - n_sent = rte_eth_tx_burst (xd->device_index, queue_id, mb, n_left); + n_sent = rte_eth_tx_burst (xd->port_id, queue_id, mb, n_left); } else { @@ -248,8 +204,8 @@ static_always_inline xd->hw_if_index)->tx_node_index; vlib_error_count (vm, node_index, DPDK_TX_FUNC_ERROR_BAD_RETVAL, 1); - clib_warning ("rte_eth_tx_burst[%d]: error %d", xd->device_index, - n_sent); + clib_warning ("rte_eth_tx_burst[%d]: error %d", + xd->port_id, n_sent); return n_left; // untransmitted packets } n_left -= n_sent; @@ -268,29 +224,6 @@ dpdk_prefetch_buffer (vlib_main_t * vm, struct rte_mbuf *mb) CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); } -static_always_inline void -dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp) -{ - dpdk_main_t *dm = &dpdk_main; - struct rte_mbuf *mb_new; - - if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0) - return; - - mb_new = dpdk_replicate_packet_mb (b); - if (PREDICT_FALSE (mb_new == 0)) - { - vlib_error_count (vm, node->node_index, - DPDK_TX_FUNC_ERROR_REPL_FAIL, 1); - b->flags |= VLIB_BUFFER_REPL_FAIL; - } - else - *mbp = mb_new; - - vec_add1 (dm->recycle[vm->thread_index], bi); -} - static_always_inline void dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b, struct rte_mbuf *mb) @@ -327,10 +260,9 @@ dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b, * node. It first copies packets on the frame to a per-thread arrays * containing the rte_mbuf pointers. */ -uword -CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * f) +VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * f) { dpdk_main_t *dm = &dpdk_main; vnet_interface_output_runtime_t *rd = (void *) node->runtime_data; @@ -345,74 +277,33 @@ CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm, thread_index); struct rte_mbuf **mb; vlib_buffer_t *b[4]; -#ifdef CLIB_HAVE_VEC256 - u64x4 off4 = u64x4_splat (buffer_main.buffer_mem_start - - sizeof (struct rte_mbuf)); - u32x8 permute_mask = { 0, 4, 1, 5, 2, 6, 3, 7 }; - u32x8 zero = { 0 }; -#endif from = vlib_frame_vector_args (f); ASSERT (n_packets <= VLIB_FRAME_SIZE); /* TX PCAP tracing */ - if (PREDICT_FALSE (dm->tx_pcap_enable)) + if (PREDICT_FALSE (dm->pcap[VLIB_TX].pcap_enable)) { n_left = n_packets; while (n_left > 0) { u32 bi0 = from[0]; vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); - if (dm->pcap_sw_if_index == 0 || - dm->pcap_sw_if_index == vnet_buffer (b0)->sw_if_index[VLIB_TX]) - pcap_add_buffer (&dm->pcap_main, vm, bi0, 512); + if (dm->pcap[VLIB_TX].pcap_sw_if_index == 0 || + dm->pcap[VLIB_TX].pcap_sw_if_index + == vnet_buffer (b0)->sw_if_index[VLIB_TX]) + pcap_add_buffer (&dm->pcap[VLIB_TX].pcap_main, vm, bi0, 512); from++; n_left--; } } /* calculate rte_mbuf pointers out of buffer indices */ - from = vlib_frame_vector_args (f); - n_left = n_packets; - mb = ptd->mbufs; - while (n_left >= 8) - { -#ifdef CLIB_HAVE_VEC256 - u32x8 bi0, bi1; - u64x4 mb0, mb1; - /* load 4 bufer indices into lower part of 256-bit register */ - bi0 = u32x8_insert_lo (zero, u32x4_load_unaligned (from)); - bi1 = u32x8_insert_lo (zero, u32x4_load_unaligned (from + 4)); - /* permute 256-bit register so each buffer index is in own u64 */ - mb0 = (u64x4) u32x8_permute (bi0, permute_mask); - mb1 = (u64x4) u32x8_permute (bi1, permute_mask); - /* shift and add to get rte_mbuf pointer */ - mb0 <<= CLIB_LOG2_CACHE_LINE_BYTES; - mb1 <<= CLIB_LOG2_CACHE_LINE_BYTES; - u64x4_store_unaligned (mb0 + off4, mb); - u64x4_store_unaligned (mb1 + off4, mb + 4); -#else - mb[0] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[0])); - mb[1] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[1])); - mb[2] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[2])); - mb[3] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[3])); - mb[4] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[4])); - mb[5] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[5])); - mb[6] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[6])); - mb[7] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[7])); -#endif - from += 8; - mb += 8; - n_left -= 8; - } - while (n_left) - { - mb[0] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[0])); - from++; - mb++; - n_left--; - } + vlib_get_buffers_with_offset (vm, vlib_frame_vector_args (f), + (void **) ptd->mbufs, n_packets, + -(i32) sizeof (struct rte_mbuf)); + from = vlib_frame_vector_args (f); n_left = n_packets; mb = ptd->mbufs; @@ -498,29 +389,6 @@ CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm, n_left--; } - /* run inly if we have buffers to recycle */ - if (PREDICT_FALSE (all_or_flags & VLIB_BUFFER_RECYCLE)) - { - struct rte_mbuf **mb_old; - from = vlib_frame_vector_args (f); - n_left = n_packets; - mb_old = mb = ptd->mbufs; - while (n_left > 0) - { - b[0] = vlib_buffer_from_rte_mbuf (mb[0]); - dpdk_buffer_recycle (vm, node, b[0], from[0], &mb_old[0]); - - /* in case of REPL_FAIL we need to shift data */ - mb[0] = mb_old[0]; - - if (PREDICT_TRUE ((b[0]->flags & VLIB_BUFFER_REPL_FAIL) == 0)) - mb++; - mb_old++; - from++; - n_left--; - } - } - /* transmit as many packets as possible */ tx_pkts = n_packets = mb - ptd->mbufs; n_left = tx_burst_vector_internal (vm, xd, ptd->mbufs, n_packets); @@ -547,18 +415,9 @@ CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm, } } - /* Recycle replicated buffers */ - if (PREDICT_FALSE (vec_len (dm->recycle[thread_index]))) - { - vlib_buffer_free (vm, dm->recycle[thread_index], - vec_len (dm->recycle[thread_index])); - _vec_len (dm->recycle[thread_index]) = 0; - } - return tx_pkts; } -#ifndef CLIB_MULTIARCH_VARIANT static void dpdk_clear_hw_interface_counters (u32 instance) { @@ -670,25 +529,25 @@ dpdk_subif_add_del_function (vnet_main_t * vnm, goto done; } - vlan_offload = rte_eth_dev_get_vlan_offload (xd->device_index); + vlan_offload = rte_eth_dev_get_vlan_offload (xd->port_id); vlan_offload |= ETH_VLAN_FILTER_OFFLOAD; - if ((r = rte_eth_dev_set_vlan_offload (xd->device_index, vlan_offload))) + if ((r = rte_eth_dev_set_vlan_offload (xd->port_id, vlan_offload))) { xd->num_subifs = prev_subifs; err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d", - xd->device_index, r); + xd->port_id, r); goto done; } if ((r = - rte_eth_dev_vlan_filter (xd->device_index, t->sub.eth.outer_vlan_id, - is_add))) + rte_eth_dev_vlan_filter (xd->port_id, + t->sub.eth.outer_vlan_id, is_add))) { xd->num_subifs = prev_subifs; err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d", - xd->device_index, r); + xd->port_id, r); goto done; } @@ -704,7 +563,6 @@ done: /* *INDENT-OFF* */ VNET_DEVICE_CLASS (dpdk_device_class) = { .name = "dpdk", - .tx_function = dpdk_interface_tx, .tx_function_n_errors = DPDK_TX_FUNC_N_ERROR, .tx_function_error_strings = dpdk_tx_func_error_strings, .format_device_name = format_dpdk_device_name, @@ -720,24 +578,9 @@ VNET_DEVICE_CLASS (dpdk_device_class) = { }; /* *INDENT-ON* */ -#if __x86_64__ -vlib_node_function_t __clib_weak dpdk_interface_tx_avx512; -vlib_node_function_t __clib_weak dpdk_interface_tx_avx2; -static void __clib_constructor -dpdk_interface_tx_multiarch_select (void) -{ - if (dpdk_interface_tx_avx512 && clib_cpu_supports_avx512f ()) - dpdk_device_class.tx_function = dpdk_interface_tx_avx512; - else if (dpdk_interface_tx_avx2 && clib_cpu_supports_avx2 ()) - dpdk_device_class.tx_function = dpdk_interface_tx_avx2; -} -#endif -#endif - #define UP_DOWN_FLAG_EVENT 1 -#ifndef CLIB_MULTIARCH_VARIANT -uword +static uword admin_up_down_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) { @@ -781,14 +624,13 @@ admin_up_down_process (vlib_main_t * vm, } /* *INDENT-OFF* */ -VLIB_REGISTER_NODE (admin_up_down_process_node,static) = { +VLIB_REGISTER_NODE (admin_up_down_process_node) = { .function = admin_up_down_process, .type = VLIB_NODE_TYPE_PROCESS, .name = "admin-up-down-process", .process_log2_n_stack_bytes = 17, // 256KB }; /* *INDENT-ON* */ -#endif /* * fd.io coding-style-patch-verification: ON