X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fdevice%2Fdevice.c;h=6285d93ac9605cefe7ce9459f10c73092d7aa4d0;hb=refs%2Fchanges%2F28%2F17828%2F2;hp=cdd9785a3f375df25d461c4030e9a3e425a71b49;hpb=a35cc14d37466f0737fa928d25697fbfe6e7d657;p=vpp.git diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index cdd9785a3f3..6285d93ac96 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -19,15 +19,14 @@ #include #include +#include #include - #include #include #define foreach_dpdk_tx_func_error \ _(BAD_RETVAL, "DPDK tx function returned an error") \ - _(PKT_DROP, "Tx packet drops (dpdk tx failure)") \ - _(REPL_FAIL, "Tx packet drops (replication failure)") + _(PKT_DROP, "Tx packet drops (dpdk tx failure)") typedef enum { @@ -37,7 +36,6 @@ typedef enum DPDK_TX_FUNC_N_ERROR, } dpdk_tx_func_error_t; -#ifndef CLIB_MULTIARCH_VARIANT static char *dpdk_tx_func_error_strings[] = { #define _(n,s) s, foreach_dpdk_tx_func_error @@ -45,13 +43,14 @@ static char *dpdk_tx_func_error_strings[] = { }; static clib_error_t * -dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address) +dpdk_set_mac_address (vnet_hw_interface_t * hi, + const u8 * old_address, const u8 * address) { int error; dpdk_main_t *dm = &dpdk_main; dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance); - error = rte_eth_dev_default_mac_addr_set (xd->device_index, + error = rte_eth_dev_default_mac_addr_set (xd->port_id, (struct ether_addr *) address); if (error) @@ -65,49 +64,6 @@ dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address) return NULL; } } -#endif - -static struct rte_mbuf * -dpdk_replicate_packet_mb (vlib_buffer_t * b) -{ - dpdk_main_t *dm = &dpdk_main; - struct rte_mbuf **mbufs = 0, *s, *d; - u8 nb_segs; - unsigned socket_id = rte_socket_id (); - int i; - - ASSERT (dm->pktmbuf_pools[socket_id]); - s = rte_mbuf_from_vlib_buffer (b); - nb_segs = s->nb_segs; - vec_validate (mbufs, nb_segs - 1); - - if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs)) - { - vec_free (mbufs); - return 0; - } - - d = mbufs[0]; - d->nb_segs = s->nb_segs; - d->data_len = s->data_len; - d->pkt_len = s->pkt_len; - d->data_off = s->data_off; - clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len); - - for (i = 1; i < nb_segs; i++) - { - d->next = mbufs[i]; - d = mbufs[i]; - s = s->next; - d->data_len = s->data_len; - clib_memcpy (d->buf_addr, s->buf_addr, - RTE_PKTMBUF_HEADROOM + s->data_len); - } - - d = mbufs[0]; - vec_free (mbufs); - return d; -} static void dpdk_tx_trace_buffer (dpdk_main_t * dm, vlib_node_runtime_t * node, @@ -124,12 +80,13 @@ dpdk_tx_trace_buffer (dpdk_main_t * dm, vlib_node_runtime_t * node, t0->queue_index = queue_id; t0->device_index = xd->device_index; t0->buffer_index = vlib_get_buffer_index (vm, buffer); - clib_memcpy (&t0->mb, mb, sizeof (t0->mb)); - clib_memcpy (&t0->buffer, buffer, - sizeof (buffer[0]) - sizeof (buffer->pre_data)); - clib_memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data, - sizeof (t0->buffer.pre_data)); - clib_memcpy (&t0->data, mb->buf_addr + mb->data_off, sizeof (t0->data)); + clib_memcpy_fast (&t0->mb, mb, sizeof (t0->mb)); + clib_memcpy_fast (&t0->buffer, buffer, + sizeof (buffer[0]) - sizeof (buffer->pre_data)); + clib_memcpy_fast (t0->buffer.pre_data, buffer->data + buffer->current_data, + sizeof (t0->buffer.pre_data)); + clib_memcpy_fast (&t0->data, mb->buf_addr + mb->data_off, + sizeof (t0->data)); } static_always_inline void @@ -170,11 +127,9 @@ dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b, mb->pkt_len = b->current_length; mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data; first_mb->nb_segs++; - if (PREDICT_FALSE (b->n_add_refs)) - { - rte_mbuf_refcnt_update (mb, b->n_add_refs); - b->n_add_refs = 0; - } + if (PREDICT_FALSE (b->ref_count > 1)) + mb->pool = + dpdk_no_cache_mempool_by_buffer_pool_index[b->buffer_pool_index]; } } @@ -206,11 +161,12 @@ static_always_inline if (PREDICT_FALSE (xd->lockp != 0)) { queue_id = queue_id % xd->tx_q_used; - while (__sync_lock_test_and_set (xd->lockp[queue_id], 1)) + while (clib_atomic_test_and_set (xd->lockp[queue_id])) /* zzzz */ queue_id = (queue_id + 1) % xd->tx_q_used; } +#if 0 if (PREDICT_FALSE (xd->flags & DPDK_DEVICE_FLAG_HQOS)) /* HQoS ON */ { /* no wrap, transmit in one burst */ @@ -223,10 +179,12 @@ static_always_inline n_sent = rte_ring_sp_enqueue_burst (hqos->swq, (void **) mb, n_left, 0); } - else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD)) + else +#endif + if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD)) { /* no wrap, transmit in one burst */ - n_sent = rte_eth_tx_burst (xd->device_index, queue_id, mb, n_left); + n_sent = rte_eth_tx_burst (xd->port_id, queue_id, mb, n_left); } else { @@ -235,7 +193,7 @@ static_always_inline } if (PREDICT_FALSE (xd->lockp != 0)) - *xd->lockp[queue_id] = 0; + clib_atomic_release (xd->lockp[queue_id]); if (PREDICT_FALSE (n_sent < 0)) { @@ -248,8 +206,8 @@ static_always_inline xd->hw_if_index)->tx_node_index; vlib_error_count (vm, node_index, DPDK_TX_FUNC_ERROR_BAD_RETVAL, 1); - clib_warning ("rte_eth_tx_burst[%d]: error %d", xd->device_index, - n_sent); + clib_warning ("rte_eth_tx_burst[%d]: error %d", + xd->port_id, n_sent); return n_left; // untransmitted packets } n_left -= n_sent; @@ -268,29 +226,6 @@ dpdk_prefetch_buffer (vlib_main_t * vm, struct rte_mbuf *mb) CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); } -static_always_inline void -dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp) -{ - dpdk_main_t *dm = &dpdk_main; - struct rte_mbuf *mb_new; - - if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0) - return; - - mb_new = dpdk_replicate_packet_mb (b); - if (PREDICT_FALSE (mb_new == 0)) - { - vlib_error_count (vm, node->node_index, - DPDK_TX_FUNC_ERROR_REPL_FAIL, 1); - b->flags |= VLIB_BUFFER_REPL_FAIL; - } - else - *mbp = mb_new; - - vec_add1 (dm->recycle[vm->thread_index], bi); -} - static_always_inline void dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b, struct rte_mbuf *mb) @@ -327,17 +262,15 @@ dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b, * node. It first copies packets on the frame to a per-thread arrays * containing the rte_mbuf pointers. */ -uword -CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * f) +VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * f) { dpdk_main_t *dm = &dpdk_main; vnet_interface_output_runtime_t *rd = (void *) node->runtime_data; dpdk_device_t *xd = vec_elt_at_index (dm->devices, rd->dev_instance); u32 n_packets = f->n_vectors; u32 n_left; - u32 *from; u32 thread_index = vm->thread_index; int queue_id = thread_index; u32 tx_pkts = 0, all_or_flags = 0; @@ -345,75 +278,14 @@ CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm, thread_index); struct rte_mbuf **mb; vlib_buffer_t *b[4]; -#ifdef CLIB_HAVE_VEC256 - u64x4 off4 = u64x4_splat (buffer_main.buffer_mem_start - - sizeof (struct rte_mbuf)); - u32x8 permute_mask = { 0, 4, 1, 5, 2, 6, 3, 7 }; - u32x8 zero = { 0 }; -#endif - - from = vlib_frame_vector_args (f); ASSERT (n_packets <= VLIB_FRAME_SIZE); - /* TX PCAP tracing */ - if (PREDICT_FALSE (dm->tx_pcap_enable)) - { - n_left = n_packets; - while (n_left > 0) - { - u32 bi0 = from[0]; - vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); - if (dm->pcap_sw_if_index == 0 || - dm->pcap_sw_if_index == vnet_buffer (b0)->sw_if_index[VLIB_TX]) - pcap_add_buffer (&dm->pcap_main, vm, bi0, 512); - from++; - n_left--; - } - } - /* calculate rte_mbuf pointers out of buffer indices */ - from = vlib_frame_vector_args (f); - n_left = n_packets; - mb = ptd->mbufs; - while (n_left >= 8) - { -#ifdef CLIB_HAVE_VEC256 - u32x8 bi0, bi1; - u64x4 mb0, mb1; - /* load 4 bufer indices into lower part of 256-bit register */ - bi0 = u32x8_insert_lo (zero, u32x4_load_unaligned (from)); - bi1 = u32x8_insert_lo (zero, u32x4_load_unaligned (from + 4)); - /* permute 256-bit register so each buffer index is in own u64 */ - mb0 = (u64x4) u32x8_permute (bi0, permute_mask); - mb1 = (u64x4) u32x8_permute (bi1, permute_mask); - /* shift and add to get rte_mbuf pointer */ - mb0 <<= CLIB_LOG2_CACHE_LINE_BYTES; - mb1 <<= CLIB_LOG2_CACHE_LINE_BYTES; - u64x4_store_unaligned (mb0 + off4, mb); - u64x4_store_unaligned (mb1 + off4, mb + 4); -#else - mb[0] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[0])); - mb[1] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[1])); - mb[2] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[2])); - mb[3] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[3])); - mb[4] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[4])); - mb[5] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[5])); - mb[6] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[6])); - mb[7] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[7])); -#endif - from += 8; - mb += 8; - n_left -= 8; - } - while (n_left) - { - mb[0] = rte_mbuf_from_vlib_buffer (vlib_get_buffer (vm, from[0])); - from++; - mb++; - n_left--; - } - from = vlib_frame_vector_args (f); + vlib_get_buffers_with_offset (vm, vlib_frame_vector_args (f), + (void **) ptd->mbufs, n_packets, + -(i32) sizeof (struct rte_mbuf)); + n_left = n_packets; mb = ptd->mbufs; @@ -498,29 +370,6 @@ CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm, n_left--; } - /* run inly if we have buffers to recycle */ - if (PREDICT_FALSE (all_or_flags & VLIB_BUFFER_RECYCLE)) - { - struct rte_mbuf **mb_old; - from = vlib_frame_vector_args (f); - n_left = n_packets; - mb_old = mb = ptd->mbufs; - while (n_left > 0) - { - b[0] = vlib_buffer_from_rte_mbuf (mb[0]); - dpdk_buffer_recycle (vm, node, b[0], from[0], &mb_old[0]); - - /* in case of REPL_FAIL we need to shift data */ - mb[0] = mb_old[0]; - - if (PREDICT_TRUE ((b[0]->flags & VLIB_BUFFER_REPL_FAIL) == 0)) - mb++; - mb_old++; - from++; - n_left--; - } - } - /* transmit as many packets as possible */ tx_pkts = n_packets = mb - ptd->mbufs; n_left = tx_burst_vector_internal (vm, xd, ptd->mbufs, n_packets); @@ -547,18 +396,9 @@ CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm, } } - /* Recycle replicated buffers */ - if (PREDICT_FALSE (vec_len (dm->recycle[thread_index]))) - { - vlib_buffer_free (vm, dm->recycle[thread_index], - vec_len (dm->recycle[thread_index])); - _vec_len (dm->recycle[thread_index]) = 0; - } - return tx_pkts; } -#ifndef CLIB_MULTIARCH_VARIANT static void dpdk_clear_hw_interface_counters (u32 instance) { @@ -571,10 +411,10 @@ dpdk_clear_hw_interface_counters (u32 instance) */ dpdk_update_counters (xd, vlib_time_now (dm->vlib_main)); - clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats)); - clib_memcpy (xd->last_cleared_xstats, xd->xstats, - vec_len (xd->last_cleared_xstats) * - sizeof (xd->last_cleared_xstats[0])); + clib_memcpy_fast (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats)); + clib_memcpy_fast (xd->last_cleared_xstats, xd->xstats, + vec_len (xd->last_cleared_xstats) * + sizeof (xd->last_cleared_xstats[0])); } @@ -591,8 +431,6 @@ dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) if (is_up) { - vnet_hw_interface_set_flags (vnm, xd->hw_if_index, - VNET_HW_INTERFACE_FLAG_LINK_UP); if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0) dpdk_device_start (xd); xd->flags |= DPDK_DEVICE_FLAG_ADMIN_UP; @@ -670,25 +508,25 @@ dpdk_subif_add_del_function (vnet_main_t * vnm, goto done; } - vlan_offload = rte_eth_dev_get_vlan_offload (xd->device_index); + vlan_offload = rte_eth_dev_get_vlan_offload (xd->port_id); vlan_offload |= ETH_VLAN_FILTER_OFFLOAD; - if ((r = rte_eth_dev_set_vlan_offload (xd->device_index, vlan_offload))) + if ((r = rte_eth_dev_set_vlan_offload (xd->port_id, vlan_offload))) { xd->num_subifs = prev_subifs; err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d", - xd->device_index, r); + xd->port_id, r); goto done; } if ((r = - rte_eth_dev_vlan_filter (xd->device_index, t->sub.eth.outer_vlan_id, - is_add))) + rte_eth_dev_vlan_filter (xd->port_id, + t->sub.eth.outer_vlan_id, is_add))) { xd->num_subifs = prev_subifs; err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d", - xd->device_index, r); + xd->port_id, r); goto done; } @@ -704,7 +542,6 @@ done: /* *INDENT-OFF* */ VNET_DEVICE_CLASS (dpdk_device_class) = { .name = "dpdk", - .tx_function = dpdk_interface_tx, .tx_function_n_errors = DPDK_TX_FUNC_N_ERROR, .tx_function_error_strings = dpdk_tx_func_error_strings, .format_device_name = format_dpdk_device_name, @@ -720,24 +557,9 @@ VNET_DEVICE_CLASS (dpdk_device_class) = { }; /* *INDENT-ON* */ -#if __x86_64__ -vlib_node_function_t __clib_weak dpdk_interface_tx_avx512; -vlib_node_function_t __clib_weak dpdk_interface_tx_avx2; -static void __clib_constructor -dpdk_interface_tx_multiarch_select (void) -{ - if (dpdk_interface_tx_avx512 && clib_cpu_supports_avx512f ()) - dpdk_device_class.tx_function = dpdk_interface_tx_avx512; - else if (dpdk_interface_tx_avx2 && clib_cpu_supports_avx2 ()) - dpdk_device_class.tx_function = dpdk_interface_tx_avx2; -} -#endif -#endif - #define UP_DOWN_FLAG_EVENT 1 -#ifndef CLIB_MULTIARCH_VARIANT -uword +static uword admin_up_down_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) { @@ -781,14 +603,13 @@ admin_up_down_process (vlib_main_t * vm, } /* *INDENT-OFF* */ -VLIB_REGISTER_NODE (admin_up_down_process_node,static) = { +VLIB_REGISTER_NODE (admin_up_down_process_node) = { .function = admin_up_down_process, .type = VLIB_NODE_TYPE_PROCESS, .name = "admin-up-down-process", .process_log2_n_stack_bytes = 17, // 256KB }; /* *INDENT-ON* */ -#endif /* * fd.io coding-style-patch-verification: ON