X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;ds=sidebyside;f=src%2Fplugins%2Fdpdk%2Fdevice%2Fdevice.c;h=8778d71cd7a3e8c02c20246c6747616285e8f289;hb=08ac303e43492c8b25911340fb62811289dd3935;hp=b2c87c7930d368360a8c1bf22cba63638aac21cf;hpb=6e36351faf5b69a0bfb8235b3b06f8b2c24e5547;p=vpp.git diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index b2c87c7930d..8778d71cd7a 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -19,15 +19,14 @@ #include #include +#include #include - #include #include #define foreach_dpdk_tx_func_error \ _(BAD_RETVAL, "DPDK tx function returned an error") \ - _(PKT_DROP, "Tx packet drops (dpdk tx failure)") \ - _(REPL_FAIL, "Tx packet drops (replication failure)") + _(PKT_DROP, "Tx packet drops (dpdk tx failure)") typedef enum { @@ -44,7 +43,8 @@ static char *dpdk_tx_func_error_strings[] = { }; static clib_error_t * -dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address) +dpdk_set_mac_address (vnet_hw_interface_t * hi, + const u8 * old_address, const u8 * address) { int error; dpdk_main_t *dm = &dpdk_main; @@ -65,48 +65,6 @@ dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address) } } -static struct rte_mbuf * -dpdk_replicate_packet_mb (vlib_buffer_t * b) -{ - dpdk_main_t *dm = &dpdk_main; - struct rte_mbuf **mbufs = 0, *s, *d; - u8 nb_segs; - unsigned socket_id = rte_socket_id (); - int i; - - ASSERT (dm->pktmbuf_pools[socket_id]); - s = rte_mbuf_from_vlib_buffer (b); - nb_segs = s->nb_segs; - vec_validate (mbufs, nb_segs - 1); - - if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs)) - { - vec_free (mbufs); - return 0; - } - - d = mbufs[0]; - d->nb_segs = s->nb_segs; - d->data_len = s->data_len; - d->pkt_len = s->pkt_len; - d->data_off = s->data_off; - clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len); - - for (i = 1; i < nb_segs; i++) - { - d->next = mbufs[i]; - d = mbufs[i]; - s = s->next; - d->data_len = s->data_len; - clib_memcpy (d->buf_addr, s->buf_addr, - RTE_PKTMBUF_HEADROOM + s->data_len); - } - - d = mbufs[0]; - vec_free (mbufs); - return d; -} - static void dpdk_tx_trace_buffer (dpdk_main_t * dm, vlib_node_runtime_t * node, dpdk_device_t * xd, u16 queue_id, @@ -122,12 +80,13 @@ dpdk_tx_trace_buffer (dpdk_main_t * dm, vlib_node_runtime_t * node, t0->queue_index = queue_id; t0->device_index = xd->device_index; t0->buffer_index = vlib_get_buffer_index (vm, buffer); - clib_memcpy (&t0->mb, mb, sizeof (t0->mb)); - clib_memcpy (&t0->buffer, buffer, - sizeof (buffer[0]) - sizeof (buffer->pre_data)); - clib_memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data, - sizeof (t0->buffer.pre_data)); - clib_memcpy (&t0->data, mb->buf_addr + mb->data_off, sizeof (t0->data)); + clib_memcpy_fast (&t0->mb, mb, sizeof (t0->mb)); + clib_memcpy_fast (&t0->buffer, buffer, + sizeof (buffer[0]) - sizeof (buffer->pre_data)); + clib_memcpy_fast (t0->buffer.pre_data, buffer->data + buffer->current_data, + sizeof (t0->buffer.pre_data)); + clib_memcpy_fast (&t0->data, mb->buf_addr + mb->data_off, + sizeof (t0->data)); } static_always_inline void @@ -135,23 +94,13 @@ dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b, int maybe_multiseg) { struct rte_mbuf *mb, *first_mb, *last_mb; + last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b); /* buffer is coming from non-dpdk source so we need to init rte_mbuf header */ if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_EXT_HDR_VALID) == 0)) - { - vlib_buffer_t *b2 = b; - last_mb = mb = rte_mbuf_from_vlib_buffer (b2); - rte_pktmbuf_reset (mb); - while (maybe_multiseg && (b2->flags & VLIB_BUFFER_NEXT_PRESENT)) - { - b2 = vlib_get_buffer (vm, b2->next_buffer); - mb = rte_mbuf_from_vlib_buffer (b2); - rte_pktmbuf_reset (mb); - } - } + rte_pktmbuf_reset (mb); - last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b); first_mb->nb_segs = 1; mb->data_len = b->current_length; mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) : @@ -162,17 +111,17 @@ dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b, { b = vlib_get_buffer (vm, b->next_buffer); mb = rte_mbuf_from_vlib_buffer (b); + if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_EXT_HDR_VALID) == 0)) + rte_pktmbuf_reset (mb); last_mb->next = mb; last_mb = mb; mb->data_len = b->current_length; mb->pkt_len = b->current_length; mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data; first_mb->nb_segs++; - if (PREDICT_FALSE (b->n_add_refs)) - { - rte_mbuf_refcnt_update (mb, b->n_add_refs); - b->n_add_refs = 0; - } + if (PREDICT_FALSE (b->ref_count > 1)) + mb->pool = + dpdk_no_cache_mempool_by_buffer_pool_index[b->buffer_pool_index]; } } @@ -204,11 +153,12 @@ static_always_inline if (PREDICT_FALSE (xd->lockp != 0)) { queue_id = queue_id % xd->tx_q_used; - while (__sync_lock_test_and_set (xd->lockp[queue_id], 1)) + while (clib_atomic_test_and_set (xd->lockp[queue_id])) /* zzzz */ queue_id = (queue_id + 1) % xd->tx_q_used; } +#if 0 if (PREDICT_FALSE (xd->flags & DPDK_DEVICE_FLAG_HQOS)) /* HQoS ON */ { /* no wrap, transmit in one burst */ @@ -221,7 +171,9 @@ static_always_inline n_sent = rte_ring_sp_enqueue_burst (hqos->swq, (void **) mb, n_left, 0); } - else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD)) + else +#endif + if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD)) { /* no wrap, transmit in one burst */ n_sent = rte_eth_tx_burst (xd->port_id, queue_id, mb, n_left); @@ -233,7 +185,7 @@ static_always_inline } if (PREDICT_FALSE (xd->lockp != 0)) - *xd->lockp[queue_id] = 0; + clib_atomic_release (xd->lockp[queue_id]); if (PREDICT_FALSE (n_sent < 0)) { @@ -262,33 +214,10 @@ static_always_inline void dpdk_prefetch_buffer (vlib_main_t * vm, struct rte_mbuf *mb) { vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb); - CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (mb, sizeof (struct rte_mbuf), STORE); CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); } -static_always_inline void -dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp) -{ - dpdk_main_t *dm = &dpdk_main; - struct rte_mbuf *mb_new; - - if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0) - return; - - mb_new = dpdk_replicate_packet_mb (b); - if (PREDICT_FALSE (mb_new == 0)) - { - vlib_error_count (vm, node->node_index, - DPDK_TX_FUNC_ERROR_REPL_FAIL, 1); - b->flags |= VLIB_BUFFER_REPL_FAIL; - } - else - *mbp = mb_new; - - vec_add1 (dm->recycle[vm->thread_index], bi); -} - static_always_inline void dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b, struct rte_mbuf *mb) @@ -334,7 +263,6 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, dpdk_device_t *xd = vec_elt_at_index (dm->devices, rd->dev_instance); u32 n_packets = f->n_vectors; u32 n_left; - u32 *from; u32 thread_index = vm->thread_index; int queue_id = thread_index; u32 tx_pkts = 0, all_or_flags = 0; @@ -343,33 +271,13 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, struct rte_mbuf **mb; vlib_buffer_t *b[4]; - from = vlib_frame_vector_args (f); - ASSERT (n_packets <= VLIB_FRAME_SIZE); - /* TX PCAP tracing */ - if (PREDICT_FALSE (dm->pcap[VLIB_TX].pcap_enable)) - { - n_left = n_packets; - while (n_left > 0) - { - u32 bi0 = from[0]; - vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); - if (dm->pcap[VLIB_TX].pcap_sw_if_index == 0 || - dm->pcap[VLIB_TX].pcap_sw_if_index - == vnet_buffer (b0)->sw_if_index[VLIB_TX]) - pcap_add_buffer (&dm->pcap[VLIB_TX].pcap_main, vm, bi0, 512); - from++; - n_left--; - } - } - /* calculate rte_mbuf pointers out of buffer indices */ vlib_get_buffers_with_offset (vm, vlib_frame_vector_args (f), (void **) ptd->mbufs, n_packets, -(i32) sizeof (struct rte_mbuf)); - from = vlib_frame_vector_args (f); n_left = n_packets; mb = ptd->mbufs; @@ -454,29 +362,6 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, n_left--; } - /* run inly if we have buffers to recycle */ - if (PREDICT_FALSE (all_or_flags & VLIB_BUFFER_RECYCLE)) - { - struct rte_mbuf **mb_old; - from = vlib_frame_vector_args (f); - n_left = n_packets; - mb_old = mb = ptd->mbufs; - while (n_left > 0) - { - b[0] = vlib_buffer_from_rte_mbuf (mb[0]); - dpdk_buffer_recycle (vm, node, b[0], from[0], &mb_old[0]); - - /* in case of REPL_FAIL we need to shift data */ - mb[0] = mb_old[0]; - - if (PREDICT_TRUE ((b[0]->flags & VLIB_BUFFER_REPL_FAIL) == 0)) - mb++; - mb_old++; - from++; - n_left--; - } - } - /* transmit as many packets as possible */ tx_pkts = n_packets = mb - ptd->mbufs; n_left = tx_burst_vector_internal (vm, xd, ptd->mbufs, n_packets); @@ -503,14 +388,6 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, } } - /* Recycle replicated buffers */ - if (PREDICT_FALSE (vec_len (dm->recycle[thread_index]))) - { - vlib_buffer_free (vm, dm->recycle[thread_index], - vec_len (dm->recycle[thread_index])); - _vec_len (dm->recycle[thread_index]) = 0; - } - return tx_pkts; } @@ -520,17 +397,8 @@ dpdk_clear_hw_interface_counters (u32 instance) dpdk_main_t *dm = &dpdk_main; dpdk_device_t *xd = vec_elt_at_index (dm->devices, instance); - /* - * Set the "last_cleared_stats" to the current stats, so that - * things appear to clear from a display perspective. - */ - dpdk_update_counters (xd, vlib_time_now (dm->vlib_main)); - - clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats)); - clib_memcpy (xd->last_cleared_xstats, xd->xstats, - vec_len (xd->last_cleared_xstats) * - sizeof (xd->last_cleared_xstats[0])); - + rte_eth_stats_reset (xd->port_id); + rte_eth_xstats_reset (xd->port_id); } static clib_error_t * @@ -546,8 +414,6 @@ dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) if (is_up) { - vnet_hw_interface_set_flags (vnm, xd->hw_if_index, - VNET_HW_INTERFACE_FLAG_LINK_UP); if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0) dpdk_device_start (xd); xd->flags |= DPDK_DEVICE_FLAG_ADMIN_UP;