X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=vnet%2Fvnet%2Fdevices%2Fdpdk%2Fdevice.c;h=011ec75e18be8f9858d8be47710aa2ff622b3ef9;hb=a30d40d33c308c254414835e116db1d72c204b50;hp=02703cc59b25af4e220ef2ca8c6eabf6848aaf81;hpb=9455084c7f85d5930182c4ad5e060f648b21ccfa;p=vpp.git diff --git a/vnet/vnet/devices/dpdk/device.c b/vnet/vnet/devices/dpdk/device.c index 02703cc59b2..011ec75e18b 100644 --- a/vnet/vnet/devices/dpdk/device.c +++ b/vnet/vnet/devices/dpdk/device.c @@ -87,7 +87,7 @@ static struct rte_mbuf * dpdk_replicate_packet_mb (vlib_buffer_t * b) unsigned socket_id = rte_socket_id(); ASSERT (bm->pktmbuf_pools[socket_id]); - pkt_mb = ((struct rte_mbuf *)b)-1; + pkt_mb = rte_mbuf_from_vlib_buffer(b); nb_segs = pkt_mb->nb_segs; for (nb_segs_left = nb_segs; nb_segs_left; nb_segs_left--) { @@ -135,7 +135,7 @@ static struct rte_mbuf * dpdk_replicate_packet_mb (vlib_buffer_t * b) rte_pktmbuf_data_len (new_mb) = pkt_mb->data_len; copy_bytes = pkt_mb->data_len + RTE_PKTMBUF_HEADROOM; ASSERT(copy_bytes <= pkt_mb->buf_len); - memcpy(new_mb->buf_addr, pkt_mb->buf_addr, copy_bytes); + clib_memcpy(new_mb->buf_addr, pkt_mb->buf_addr, copy_bytes); prev_mb_next = &new_mb->next; pkt_mb = pkt_mb->next; @@ -159,15 +159,15 @@ dpdk_tx_trace_buffer (dpdk_main_t * dm, dpdk_tx_dma_trace_t * t0; struct rte_mbuf * mb; - mb = ((struct rte_mbuf *)buffer)-1; + mb = rte_mbuf_from_vlib_buffer(buffer); t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0])); t0->queue_index = queue_id; t0->device_index = xd->device_index; t0->buffer_index = buffer_index; - memcpy (&t0->mb, mb, sizeof (t0->mb)); - memcpy (&t0->buffer, buffer, sizeof (buffer[0]) - sizeof (buffer->pre_data)); - memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data, + clib_memcpy (&t0->mb, mb, sizeof (t0->mb)); + clib_memcpy (&t0->buffer, buffer, sizeof (buffer[0]) - sizeof (buffer->pre_data)); + clib_memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data, sizeof (t0->buffer.pre_data)); } @@ -321,11 +321,12 @@ u32 tx_burst_vector_internal (vlib_main_t * vm, (uint16_t) (tx_head-tx_tail)); if (PREDICT_TRUE(rv > 0)) { - dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]); + dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_TXQ]); vring->packets += rv; vring->bytes += bytes; if (dpdk_vhost_user_want_interrupt(xd, offset + VIRTIO_RXQ)) { + vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]); vring->n_since_last_int += rv; f64 now = vlib_time_now (vm); @@ -358,11 +359,12 @@ u32 tx_burst_vector_internal (vlib_main_t * vm, if (PREDICT_TRUE(rv > 0)) { - dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]); + dpdk_vu_vring *vring = &(xd->vu_intf->vrings[offset + VIRTIO_TXQ]); vring->packets += rv; vring->bytes += bytes; if (dpdk_vhost_user_want_interrupt(xd, offset + VIRTIO_RXQ)) { + vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]); vring->n_since_last_int += rv; f64 now = vlib_time_now (vm); @@ -541,7 +543,7 @@ dpdk_interface_tx (vlib_main_t * vm, { u32 bi0 = from[n_packets]; vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); - struct rte_mbuf *mb0 = ((struct rte_mbuf *)b0) - 1; + struct rte_mbuf *mb0 = rte_mbuf_from_vlib_buffer(b0); rte_pktmbuf_free (mb0); } return n_on_ring; @@ -584,9 +586,9 @@ dpdk_interface_tx (vlib_main_t * vm, pref0 = vlib_get_buffer (vm, pi0); pref1 = vlib_get_buffer (vm, pi1); - prefmb0 = ((struct rte_mbuf *)pref0) - 1; - prefmb1 = ((struct rte_mbuf *)pref1) - 1; - + prefmb0 = rte_mbuf_from_vlib_buffer(pref0); + prefmb1 = rte_mbuf_from_vlib_buffer(pref1); + CLIB_PREFETCH(prefmb0, CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH(pref0, CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH(prefmb1, CLIB_CACHE_LINE_BYTES, LOAD); @@ -599,44 +601,44 @@ dpdk_interface_tx (vlib_main_t * vm, b0 = vlib_get_buffer (vm, bi0); b1 = vlib_get_buffer (vm, bi1); - mb0 = ((struct rte_mbuf *)b0) - 1; - mb1 = ((struct rte_mbuf *)b1) - 1; + mb0 = rte_mbuf_from_vlib_buffer(b0); + mb1 = rte_mbuf_from_vlib_buffer(b1); any_clone = b0->clone_count | b1->clone_count; if (PREDICT_FALSE(any_clone != 0)) { if (PREDICT_FALSE(b0->clone_count != 0)) - { - struct rte_mbuf * mb0_new = dpdk_replicate_packet_mb (b0); - if (PREDICT_FALSE(mb0_new == 0)) - { - vlib_error_count (vm, node->node_index, - DPDK_TX_FUNC_ERROR_REPL_FAIL, 1); - b0->flags |= VLIB_BUFFER_REPL_FAIL; - } - else - mb0 = mb0_new; - vec_add1 (dm->recycle[my_cpu], bi0); - } + { + struct rte_mbuf * mb0_new = dpdk_replicate_packet_mb (b0); + if (PREDICT_FALSE(mb0_new == 0)) + { + vlib_error_count (vm, node->node_index, + DPDK_TX_FUNC_ERROR_REPL_FAIL, 1); + b0->flags |= VLIB_BUFFER_REPL_FAIL; + } + else + mb0 = mb0_new; + vec_add1 (dm->recycle[my_cpu], bi0); + } if (PREDICT_FALSE(b1->clone_count != 0)) - { - struct rte_mbuf * mb1_new = dpdk_replicate_packet_mb (b1); - if (PREDICT_FALSE(mb1_new == 0)) - { - vlib_error_count (vm, node->node_index, - DPDK_TX_FUNC_ERROR_REPL_FAIL, 1); - b1->flags |= VLIB_BUFFER_REPL_FAIL; - } - else - mb1 = mb1_new; - vec_add1 (dm->recycle[my_cpu], bi1); - } - } + { + struct rte_mbuf * mb1_new = dpdk_replicate_packet_mb (b1); + if (PREDICT_FALSE(mb1_new == 0)) + { + vlib_error_count (vm, node->node_index, + DPDK_TX_FUNC_ERROR_REPL_FAIL, 1); + b1->flags |= VLIB_BUFFER_REPL_FAIL; + } + else + mb1 = mb1_new; + vec_add1 (dm->recycle[my_cpu], bi1); + } + } delta0 = PREDICT_FALSE(b0->flags & VLIB_BUFFER_REPL_FAIL) ? 0 : - vlib_buffer_length_in_chain (vm, b0) - (i16) mb0->pkt_len; + vlib_buffer_length_in_chain (vm, b0) - (i16) mb0->pkt_len; delta1 = PREDICT_FALSE(b1->flags & VLIB_BUFFER_REPL_FAIL) ? 0 : - vlib_buffer_length_in_chain (vm, b1) - (i16) mb1->pkt_len; + vlib_buffer_length_in_chain (vm, b1) - (i16) mb1->pkt_len; new_data_len0 = (u16)((i16) mb0->data_len + delta0); new_data_len1 = (u16)((i16) mb1->data_len + delta1); @@ -651,23 +653,23 @@ dpdk_interface_tx (vlib_main_t * vm, mb1->pkt_len = new_pkt_len1; mb0->data_off = (PREDICT_FALSE(b0->flags & VLIB_BUFFER_REPL_FAIL)) ? - mb0->data_off : (u16)(RTE_PKTMBUF_HEADROOM + b0->current_data); + mb0->data_off : (u16)(RTE_PKTMBUF_HEADROOM + b0->current_data); mb1->data_off = (PREDICT_FALSE(b1->flags & VLIB_BUFFER_REPL_FAIL)) ? - mb1->data_off : (u16)(RTE_PKTMBUF_HEADROOM + b1->current_data); + mb1->data_off : (u16)(RTE_PKTMBUF_HEADROOM + b1->current_data); if (PREDICT_FALSE(node->flags & VLIB_NODE_FLAG_TRACE)) - { + { if (b0->flags & VLIB_BUFFER_IS_TRACED) - dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0); + dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0); if (b1->flags & VLIB_BUFFER_IS_TRACED) - dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi1, b1); - } + dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi1, b1); + } if (PREDICT_TRUE(any_clone == 0)) { - tx_vector[i % DPDK_TX_RING_SIZE] = mb0; + tx_vector[i % DPDK_TX_RING_SIZE] = mb0; i++; - tx_vector[i % DPDK_TX_RING_SIZE] = mb1; + tx_vector[i % DPDK_TX_RING_SIZE] = mb1; i++; } else @@ -675,12 +677,12 @@ dpdk_interface_tx (vlib_main_t * vm, /* cloning was done, need to check for failure */ if (PREDICT_TRUE((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0)) { - tx_vector[i % DPDK_TX_RING_SIZE] = mb0; + tx_vector[i % DPDK_TX_RING_SIZE] = mb0; i++; } if (PREDICT_TRUE((b1->flags & VLIB_BUFFER_REPL_FAIL) == 0)) { - tx_vector[i % DPDK_TX_RING_SIZE] = mb1; + tx_vector[i % DPDK_TX_RING_SIZE] = mb1; i++; } } @@ -701,23 +703,23 @@ dpdk_interface_tx (vlib_main_t * vm, b0 = vlib_get_buffer (vm, bi0); - mb0 = ((struct rte_mbuf *)b0) - 1; + mb0 = rte_mbuf_from_vlib_buffer(b0); if (PREDICT_FALSE(b0->clone_count != 0)) - { - struct rte_mbuf * mb0_new = dpdk_replicate_packet_mb (b0); - if (PREDICT_FALSE(mb0_new == 0)) - { - vlib_error_count (vm, node->node_index, - DPDK_TX_FUNC_ERROR_REPL_FAIL, 1); - b0->flags |= VLIB_BUFFER_REPL_FAIL; - } - else - mb0 = mb0_new; - vec_add1 (dm->recycle[my_cpu], bi0); - } + { + struct rte_mbuf * mb0_new = dpdk_replicate_packet_mb (b0); + if (PREDICT_FALSE(mb0_new == 0)) + { + vlib_error_count (vm, node->node_index, + DPDK_TX_FUNC_ERROR_REPL_FAIL, 1); + b0->flags |= VLIB_BUFFER_REPL_FAIL; + } + else + mb0 = mb0_new; + vec_add1 (dm->recycle[my_cpu], bi0); + } delta0 = PREDICT_FALSE(b0->flags & VLIB_BUFFER_REPL_FAIL) ? 0 : - vlib_buffer_length_in_chain (vm, b0) - (i16) mb0->pkt_len; + vlib_buffer_length_in_chain (vm, b0) - (i16) mb0->pkt_len; new_data_len0 = (u16)((i16) mb0->data_len + delta0); new_pkt_len0 = (u16)((i16) mb0->pkt_len + delta0); @@ -726,15 +728,15 @@ dpdk_interface_tx (vlib_main_t * vm, mb0->data_len = new_data_len0; mb0->pkt_len = new_pkt_len0; mb0->data_off = (PREDICT_FALSE(b0->flags & VLIB_BUFFER_REPL_FAIL)) ? - mb0->data_off : (u16)(RTE_PKTMBUF_HEADROOM + b0->current_data); + mb0->data_off : (u16)(RTE_PKTMBUF_HEADROOM + b0->current_data); if (PREDICT_FALSE(node->flags & VLIB_NODE_FLAG_TRACE)) - if (b0->flags & VLIB_BUFFER_IS_TRACED) - dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0); + if (b0->flags & VLIB_BUFFER_IS_TRACED) + dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0); if (PREDICT_TRUE((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0)) { - tx_vector[i % DPDK_TX_RING_SIZE] = mb0; + tx_vector[i % DPDK_TX_RING_SIZE] = mb0; i++; } n_left--; @@ -781,15 +783,15 @@ dpdk_interface_tx (vlib_main_t * vm, vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, n_packets); vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_PKT_DROP, - n_packets); + n_packets); while (n_packets--) rte_pktmbuf_free (tx_vector[ring->tx_tail + n_packets]); } /* Reset head/tail to avoid unnecessary wrap */ - ring->tx_head = 0; - ring->tx_tail = 0; + ring->tx_head = 0; + ring->tx_tail = 0; } /* Recycle replicated buffers */ @@ -811,9 +813,9 @@ static int dpdk_device_renumber (vnet_hw_interface_t * hi, dpdk_device_t * xd = vec_elt_at_index (dm->devices, hi->dev_instance); if (!xd || xd->dev_type != VNET_DPDK_DEV_VHOST_USER) { - clib_warning("cannot renumber non-vhost-user interface (sw_if_index: %d)", - hi->sw_if_index); - return 0; + clib_warning("cannot renumber non-vhost-user interface (sw_if_index: %d)", + hi->sw_if_index); + return 0; } xd->vu_if_id = new_dev_instance; @@ -833,23 +835,33 @@ static void dpdk_clear_hw_interface_counters (u32 instance) */ if (xd->admin_up != 0xff) { - rte_eth_stats_reset (xd->device_index); - memset (&xd->last_stats, 0, sizeof (xd->last_stats)); + /* + * Set the "last_cleared_stats" to the current stats, so that + * things appear to clear from a display perspective. + */ dpdk_update_counters (xd, vlib_time_now (dm->vlib_main)); + + clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof(xd->stats)); + clib_memcpy (xd->last_cleared_xstats, xd->xstats, + vec_len(xd->last_cleared_xstats) * + sizeof(xd->last_cleared_xstats[0])); } else { - rte_eth_stats_reset (xd->device_index); - memset(&xd->stats, 0, sizeof(xd->stats)); + /* + * Internally rte_eth_xstats_reset() is calling rte_eth_stats_reset(), + * so we're only calling xstats_reset() here. + */ + rte_eth_xstats_reset (xd->device_index); + memset (&xd->stats, 0, sizeof(xd->stats)); memset (&xd->last_stats, 0, sizeof (xd->last_stats)); } - rte_eth_xstats_reset(xd->device_index); if (PREDICT_FALSE(xd->dev_type == VNET_DPDK_DEV_VHOST_USER)) { int i; for (i = 0; i < xd->rx_q_used * VIRTIO_QNUM; i++) { - xd->vu_intf->vrings[i].packets = 0; - xd->vu_intf->vrings[i].bytes = 0; + xd->vu_intf->vrings[i].packets = 0; + xd->vu_intf->vrings[i].bytes = 0; } } } @@ -921,7 +933,7 @@ dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) vlib_buffer_main_t * bm = vm->buffer_main; memset(&conf, 0, sizeof(conf)); snprintf(conf.name, RTE_KNI_NAMESIZE, "vpp%u", xd->kni_port_id); - conf.mbuf_size = MBUF_SIZE; + conf.mbuf_size = VLIB_BUFFER_DATA_SIZE; memset(&ops, 0, sizeof(ops)); ops.port_id = xd->kni_port_id; ops.change_mtu = kni_change_mtu; @@ -1060,8 +1072,10 @@ dpdk_subif_add_del_function (vnet_main_t * vnm, if (xd->dev_type != VNET_DPDK_DEV_ETH) return 0; - /* currently we program VLANS only for IXGBE VF */ - if (xd->pmd != VNET_DPDK_PMD_IXGBEVF) + + /* currently we program VLANS only for IXGBE VF and I40E VF */ + if ((xd->pmd != VNET_DPDK_PMD_IXGBEVF) && + (xd->pmd != VNET_DPDK_PMD_I40EVF)) return 0; if (t->sub.eth.flags.no_tags == 1) @@ -1221,3 +1235,81 @@ int rte_delay_us_override (unsigned us) { } return 0; // no override } + +/* + * Return a copy of the DPDK port stats in dest. + */ +clib_error_t* +dpdk_get_hw_interface_stats (u32 hw_if_index, struct rte_eth_stats* dest) +{ + dpdk_main_t * dm = &dpdk_main; + vnet_main_t * vnm = vnet_get_main(); + vnet_hw_interface_t * hi = vnet_get_hw_interface (vnm, hw_if_index); + dpdk_device_t * xd = vec_elt_at_index (dm->devices, hi->dev_instance); + + if (!dest) { + return clib_error_return (0, "Missing or NULL argument"); + } + if (!xd) { + return clib_error_return (0, "Unable to get DPDK device from HW interface"); + } + + dpdk_update_counters (xd, vlib_time_now (dm->vlib_main)); + + clib_memcpy(dest, &xd->stats, sizeof(xd->stats)); + return (0); +} + +/* + * Return the number of dpdk mbufs + */ +u32 dpdk_num_mbufs (void) +{ + dpdk_main_t * dm = &dpdk_main; + + return dm->num_mbufs; +} + +/* + * Return the io_thread_release + */ +int dpdk_io_thread_release (void) +{ + dpdk_main_t * dm = &dpdk_main; + + return dm->io_thread_release; +} + +/* + * Return the pmd type for a given hardware interface + */ +dpdk_pmd_t dpdk_get_pmd_type (vnet_hw_interface_t *hi) +{ + dpdk_main_t * dm = &dpdk_main; + dpdk_device_t * xd; + + assert (hi); + + xd = vec_elt_at_index (dm->devices, hi->dev_instance); + + assert (xd); + + return xd->pmd; +} + +/* + * Return the cpu socket for a given hardware interface + */ +i8 dpdk_get_cpu_socket (vnet_hw_interface_t *hi) +{ + dpdk_main_t * dm = &dpdk_main; + dpdk_device_t * xd; + + assert (hi); + + xd = vec_elt_at_index(dm->devices, hi->dev_instance); + + assert (xd); + + return xd->cpu_socket; +}