n_packets = ring->tx_head - ring->tx_tail;
- tx_head = ring->tx_head % DPDK_TX_RING_SIZE;
+ tx_head = ring->tx_head % xd->nb_tx_desc;
/*
* Ensure rte_eth_tx_burst is not called with 0 packets, which can lead to
* a bit because it decreases the probability of having to issue two tx_burst
* calls due to a ring wrap.
*/
- ASSERT (n_packets < DPDK_TX_RING_SIZE);
+ ASSERT (n_packets < xd->nb_tx_desc);
/*
* If there is no flowcontrol callback, there is only temporary buffering
do
{
/* start the burst at the tail */
- tx_tail = ring->tx_tail % DPDK_TX_RING_SIZE;
+ tx_tail = ring->tx_tail % xd->nb_tx_desc;
/*
* This device only supports one TX queue,
* and we're running multi-threaded...
*/
- if (PREDICT_FALSE (xd->dev_type != VNET_DPDK_DEV_VHOST_USER &&
- xd->lockp != 0))
+ if (PREDICT_FALSE (xd->lockp != 0))
{
queue_id = queue_id % xd->tx_q_used;
while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
queue_id = (queue_id + 1) % xd->tx_q_used;
}
- if (PREDICT_TRUE (xd->dev_type == VNET_DPDK_DEV_ETH))
+ if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_HQOS)) /* HQoS ON */
{
if (PREDICT_TRUE (tx_head > tx_tail))
{
/* no wrap, transmit in one burst */
- rv = rte_eth_tx_burst (xd->device_index,
- (uint16_t) queue_id,
- &tx_vector[tx_tail],
- (uint16_t) (tx_head - tx_tail));
+ dpdk_device_hqos_per_worker_thread_t *hqos =
+ &xd->hqos_wt[vm->cpu_index];
+
+ dpdk_hqos_metadata_set (hqos,
+ &tx_vector[tx_tail], tx_head - tx_tail);
+ rv = rte_ring_sp_enqueue_burst (hqos->swq,
+ (void **) &tx_vector[tx_tail],
+ (uint16_t) (tx_head - tx_tail));
}
else
{
* at the start of the ring.
* Transmit pkts up to the wrap point.
*/
- rv = rte_eth_tx_burst (xd->device_index,
- (uint16_t) queue_id,
- &tx_vector[tx_tail],
- (uint16_t) (DPDK_TX_RING_SIZE -
- tx_tail));
-
+ dpdk_device_hqos_per_worker_thread_t *hqos =
+ &xd->hqos_wt[vm->cpu_index];
+
+ dpdk_hqos_metadata_set (hqos,
+ &tx_vector[tx_tail],
+ xd->nb_tx_desc - tx_tail);
+ rv = rte_ring_sp_enqueue_burst (hqos->swq,
+ (void **) &tx_vector[tx_tail],
+ (uint16_t) (xd->nb_tx_desc -
+ tx_tail));
/*
* If we transmitted everything we wanted, then allow 1 retry
* so we can try to transmit the rest. If we didn't transmit
* everything, stop now.
*/
- n_retry = (rv == DPDK_TX_RING_SIZE - tx_tail) ? 1 : 0;
- }
- }
-#if DPDK_VHOST_USER
- else if (xd->dev_type == VNET_DPDK_DEV_VHOST_USER)
- {
- u32 offset = 0;
- if (xd->need_txlock)
- {
- queue_id = 0;
- while (__sync_lock_test_and_set (xd->lockp[queue_id], 1));
- }
- else
- {
- dpdk_device_and_queue_t *dq;
- vec_foreach (dq, dm->devices_by_cpu[vm->cpu_index])
- {
- if (xd->device_index == dq->device)
- break;
- }
- assert (dq);
- offset = dq->queue_id * VIRTIO_QNUM;
- }
- if (PREDICT_TRUE (tx_head > tx_tail))
- {
- int i;
- u32 bytes = 0;
- struct rte_mbuf **pkts = &tx_vector[tx_tail];
- for (i = 0; i < (tx_head - tx_tail); i++)
- {
- struct rte_mbuf *buff = pkts[i];
- bytes += rte_pktmbuf_data_len (buff);
- }
-
- /* no wrap, transmit in one burst */
- rv =
- rte_vhost_enqueue_burst (&xd->vu_vhost_dev,
- offset + VIRTIO_RXQ,
- &tx_vector[tx_tail],
- (uint16_t) (tx_head - tx_tail));
- if (PREDICT_TRUE (rv > 0))
- {
- dpdk_vu_vring *vring =
- &(xd->vu_intf->vrings[offset + VIRTIO_TXQ]);
- vring->packets += rv;
- vring->bytes += bytes;
-
- if (dpdk_vhost_user_want_interrupt
- (xd, offset + VIRTIO_RXQ))
- {
- vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]);
- vring->n_since_last_int += rv;
-
- f64 now = vlib_time_now (vm);
- if (vring->int_deadline < now ||
- vring->n_since_last_int >
- dm->conf->vhost_coalesce_frames)
- dpdk_vhost_user_send_interrupt (vm, xd,
- offset + VIRTIO_RXQ);
- }
-
- int c = rv;
- while (c--)
- rte_pktmbuf_free (tx_vector[tx_tail + c]);
- }
- }
- else
- {
- /*
- * If we transmitted everything we wanted, then allow 1 retry
- * so we can try to transmit the rest. If we didn't transmit
- * everything, stop now.
- */
- int i;
- u32 bytes = 0;
- struct rte_mbuf **pkts = &tx_vector[tx_tail];
- for (i = 0; i < (DPDK_TX_RING_SIZE - tx_tail); i++)
- {
- struct rte_mbuf *buff = pkts[i];
- bytes += rte_pktmbuf_data_len (buff);
- }
- rv =
- rte_vhost_enqueue_burst (&xd->vu_vhost_dev,
- offset + VIRTIO_RXQ,
- &tx_vector[tx_tail],
- (uint16_t) (DPDK_TX_RING_SIZE -
- tx_tail));
-
- if (PREDICT_TRUE (rv > 0))
- {
- dpdk_vu_vring *vring =
- &(xd->vu_intf->vrings[offset + VIRTIO_TXQ]);
- vring->packets += rv;
- vring->bytes += bytes;
-
- if (dpdk_vhost_user_want_interrupt
- (xd, offset + VIRTIO_RXQ))
- {
- vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]);
- vring->n_since_last_int += rv;
-
- f64 now = vlib_time_now (vm);
- if (vring->int_deadline < now ||
- vring->n_since_last_int >
- dm->conf->vhost_coalesce_frames)
- dpdk_vhost_user_send_interrupt (vm, xd,
- offset + VIRTIO_RXQ);
- }
-
- int c = rv;
- while (c--)
- rte_pktmbuf_free (tx_vector[tx_tail + c]);
- }
-
- n_retry = (rv == DPDK_TX_RING_SIZE - tx_tail) ? 1 : 0;
+ n_retry = (rv == xd->nb_tx_desc - tx_tail) ? 1 : 0;
}
-
- if (xd->need_txlock)
- *xd->lockp[queue_id] = 0;
}
-#endif
-#if RTE_LIBRTE_KNI
- else if (xd->dev_type == VNET_DPDK_DEV_KNI)
+ else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
{
if (PREDICT_TRUE (tx_head > tx_tail))
{
/* no wrap, transmit in one burst */
- rv = rte_kni_tx_burst (xd->kni,
+ rv = rte_eth_tx_burst (xd->device_index,
+ (uint16_t) queue_id,
&tx_vector[tx_tail],
(uint16_t) (tx_head - tx_tail));
}
* at the start of the ring.
* Transmit pkts up to the wrap point.
*/
- rv = rte_kni_tx_burst (xd->kni,
+ rv = rte_eth_tx_burst (xd->device_index,
+ (uint16_t) queue_id,
&tx_vector[tx_tail],
- (uint16_t) (DPDK_TX_RING_SIZE -
- tx_tail));
+ (uint16_t) (xd->nb_tx_desc - tx_tail));
/*
* If we transmitted everything we wanted, then allow 1 retry
* so we can try to transmit the rest. If we didn't transmit
* everything, stop now.
*/
- n_retry = (rv == DPDK_TX_RING_SIZE - tx_tail) ? 1 : 0;
+ n_retry = (rv == xd->nb_tx_desc - tx_tail) ? 1 : 0;
}
}
-#endif
else
{
ASSERT (0);
rv = 0;
}
- if (PREDICT_FALSE (xd->dev_type != VNET_DPDK_DEV_VHOST_USER &&
- xd->lockp != 0))
+ if (PREDICT_FALSE (xd->lockp != 0))
*xd->lockp[queue_id] = 0;
if (PREDICT_FALSE (rv < 0))
ASSERT (n_packets <= VLIB_FRAME_SIZE);
- if (PREDICT_FALSE (n_on_ring + n_packets > DPDK_TX_RING_SIZE))
+ if (PREDICT_FALSE (n_on_ring + n_packets > xd->nb_tx_desc))
{
/*
* Overflowing the ring should never happen.
from = vlib_frame_vector_args (f);
n_left = n_packets;
- i = ring->tx_head % DPDK_TX_RING_SIZE;
+ i = ring->tx_head % xd->nb_tx_desc;
while (n_left >= 4)
{
if (PREDICT_TRUE (any_clone == 0))
{
- tx_vector[i % DPDK_TX_RING_SIZE] = mb0;
+ tx_vector[i % xd->nb_tx_desc] = mb0;
i++;
- tx_vector[i % DPDK_TX_RING_SIZE] = mb1;
+ tx_vector[i % xd->nb_tx_desc] = mb1;
i++;
}
else
/* cloning was done, need to check for failure */
if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0))
{
- tx_vector[i % DPDK_TX_RING_SIZE] = mb0;
+ tx_vector[i % xd->nb_tx_desc] = mb0;
i++;
}
if (PREDICT_TRUE ((b1->flags & VLIB_BUFFER_REPL_FAIL) == 0))
{
- tx_vector[i % DPDK_TX_RING_SIZE] = mb1;
+ tx_vector[i % xd->nb_tx_desc] = mb1;
i++;
}
}
if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0))
{
- tx_vector[i % DPDK_TX_RING_SIZE] = mb0;
+ tx_vector[i % xd->nb_tx_desc] = mb0;
i++;
}
n_left--;
return tx_pkts;
}
-static int
-dpdk_device_renumber (vnet_hw_interface_t * hi, u32 new_dev_instance)
-{
-#if DPDK_VHOST_USER
- dpdk_main_t *dm = &dpdk_main;
- dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance);
-
- if (!xd || xd->dev_type != VNET_DPDK_DEV_VHOST_USER)
- {
- clib_warning
- ("cannot renumber non-vhost-user interface (sw_if_index: %d)",
- hi->sw_if_index);
- return 0;
- }
-
- xd->vu_if_id = new_dev_instance;
-#endif
- return 0;
-}
-
static void
dpdk_clear_hw_interface_counters (u32 instance)
{
dpdk_device_t *xd = vec_elt_at_index (dm->devices, instance);
/*
- * DAW-FIXME: VMXNET3 device stop/start doesn't work,
- * therefore fake the stop in the dpdk driver by
- * silently dropping all of the incoming pkts instead of
- * stopping the driver / hardware.
+ * Set the "last_cleared_stats" to the current stats, so that
+ * things appear to clear from a display perspective.
*/
- if (xd->admin_up != 0xff)
- {
- /*
- * Set the "last_cleared_stats" to the current stats, so that
- * things appear to clear from a display perspective.
- */
- dpdk_update_counters (xd, vlib_time_now (dm->vlib_main));
-
- clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats));
- clib_memcpy (xd->last_cleared_xstats, xd->xstats,
- vec_len (xd->last_cleared_xstats) *
- sizeof (xd->last_cleared_xstats[0]));
- }
- else
- {
- /*
- * Internally rte_eth_xstats_reset() is calling rte_eth_stats_reset(),
- * so we're only calling xstats_reset() here.
- */
- rte_eth_xstats_reset (xd->device_index);
- memset (&xd->stats, 0, sizeof (xd->stats));
- memset (&xd->last_stats, 0, sizeof (xd->last_stats));
- }
-
-#if DPDK_VHOST_USER
- if (PREDICT_FALSE (xd->dev_type == VNET_DPDK_DEV_VHOST_USER))
- {
- int i;
- for (i = 0; i < xd->rx_q_used * VIRTIO_QNUM; i++)
- {
- xd->vu_intf->vrings[i].packets = 0;
- xd->vu_intf->vrings[i].bytes = 0;
- }
- }
-#endif
-}
-
-#ifdef RTE_LIBRTE_KNI
-static int
-kni_config_network_if (u8 port_id, u8 if_up)
-{
- vnet_main_t *vnm = vnet_get_main ();
- dpdk_main_t *dm = &dpdk_main;
- dpdk_device_t *xd;
- uword *p;
-
- p = hash_get (dm->dpdk_device_by_kni_port_id, port_id);
- if (p == 0)
- {
- clib_warning ("unknown interface");
- return 0;
- }
- else
- {
- xd = vec_elt_at_index (dm->devices, p[0]);
- }
-
- vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index,
- if_up ? VNET_HW_INTERFACE_FLAG_LINK_UP |
- ETH_LINK_FULL_DUPLEX : 0);
- return 0;
-}
-
-static int
-kni_change_mtu (u8 port_id, unsigned new_mtu)
-{
- vnet_main_t *vnm = vnet_get_main ();
- dpdk_main_t *dm = &dpdk_main;
- dpdk_device_t *xd;
- uword *p;
- vnet_hw_interface_t *hif;
-
- p = hash_get (dm->dpdk_device_by_kni_port_id, port_id);
- if (p == 0)
- {
- clib_warning ("unknown interface");
- return 0;
- }
- else
- {
- xd = vec_elt_at_index (dm->devices, p[0]);
- }
- hif = vnet_get_hw_interface (vnm, xd->vlib_hw_if_index);
+ dpdk_update_counters (xd, vlib_time_now (dm->vlib_main));
- hif->max_packet_bytes = new_mtu;
+ clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats));
+ clib_memcpy (xd->last_cleared_xstats, xd->xstats,
+ vec_len (xd->last_cleared_xstats) *
+ sizeof (xd->last_cleared_xstats[0]));
- return 0;
}
-#endif
static clib_error_t *
dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
dpdk_device_t *xd = vec_elt_at_index (dm->devices, hif->dev_instance);
int rv = 0;
-#ifdef RTE_LIBRTE_KNI
- if (xd->dev_type == VNET_DPDK_DEV_KNI)
- {
- if (is_up)
- {
- struct rte_kni_conf conf;
- struct rte_kni_ops ops;
- vlib_main_t *vm = vlib_get_main ();
- vlib_buffer_main_t *bm = vm->buffer_main;
- memset (&conf, 0, sizeof (conf));
- snprintf (conf.name, RTE_KNI_NAMESIZE, "vpp%u", xd->kni_port_id);
- conf.mbuf_size = VLIB_BUFFER_DATA_SIZE;
- memset (&ops, 0, sizeof (ops));
- ops.port_id = xd->kni_port_id;
- ops.change_mtu = kni_change_mtu;
- ops.config_network_if = kni_config_network_if;
-
- xd->kni =
- rte_kni_alloc (bm->pktmbuf_pools[rte_socket_id ()], &conf, &ops);
- if (!xd->kni)
- {
- clib_warning ("failed to allocate kni interface");
- }
- else
- {
- hif->max_packet_bytes = 1500; /* kni interface default value */
- xd->admin_up = 1;
- }
- }
- else
- {
- xd->admin_up = 0;
- int kni_rv;
-
- kni_rv = rte_kni_release (xd->kni);
- if (kni_rv < 0)
- clib_warning ("rte_kni_release returned %d", kni_rv);
- }
- return 0;
- }
-#endif
-#if DPDK_VHOST_USER
- if (xd->dev_type == VNET_DPDK_DEV_VHOST_USER)
- {
- if (is_up)
- {
- if (xd->vu_is_running)
- vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index,
- VNET_HW_INTERFACE_FLAG_LINK_UP |
- ETH_LINK_FULL_DUPLEX);
- xd->admin_up = 1;
- }
- else
- {
- vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, 0);
- xd->admin_up = 0;
- }
-
- return 0;
- }
-#endif
-
-
if (is_up)
{
f64 now = vlib_time_now (dm->vlib_main);
- /*
- * DAW-FIXME: VMXNET3 device stop/start doesn't work,
- * therefore fake the stop in the dpdk driver by
- * silently dropping all of the incoming pkts instead of
- * stopping the driver / hardware.
- */
- if (xd->admin_up == 0)
+ if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
rv = rte_eth_dev_start (xd->device_index);
- if (xd->promisc)
+ if (xd->flags & DPDK_DEVICE_FLAG_PROMISC)
rte_eth_promiscuous_enable (xd->device_index);
else
rte_eth_promiscuous_disable (xd->device_index);
rte_eth_allmulticast_enable (xd->device_index);
- xd->admin_up = 1;
+ xd->flags |= DPDK_DEVICE_FLAG_ADMIN_UP;
dpdk_update_counters (xd, now);
dpdk_update_link_state (xd, now);
}
else
{
- /*
- * DAW-FIXME: VMXNET3 device stop/start doesn't work,
- * therefore fake the stop in the dpdk driver by
- * silently dropping all of the incoming pkts instead of
- * stopping the driver / hardware.
- */
- if (xd->pmd != VNET_DPDK_PMD_VMXNET3)
- xd->admin_up = 0;
- else
- xd->admin_up = ~0;
+ xd->flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP;
rte_eth_allmulticast_disable (xd->device_index);
vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, 0);
-
- /*
- * DAW-FIXME: VMXNET3 device stop/start doesn't work,
- * therefore fake the stop in the dpdk driver by
- * silently dropping all of the incoming pkts instead of
- * stopping the driver / hardware.
- */
- if (xd->pmd != VNET_DPDK_PMD_VMXNET3)
- rte_eth_dev_stop (xd->device_index);
+ rte_eth_dev_stop (xd->device_index);
/* For bonded interface, stop slave links */
if (xd->pmd == VNET_DPDK_PMD_BOND)
dpdk_device_t *xd = vec_elt_at_index (xm->devices, hw->dev_instance);
vnet_sw_interface_t *t = (vnet_sw_interface_t *) st;
int r, vlan_offload;
- u32 prev_subifs = xd->vlan_subifs;
+ u32 prev_subifs = xd->num_subifs;
+ clib_error_t *err = 0;
if (is_add)
- xd->vlan_subifs++;
- else if (xd->vlan_subifs)
- xd->vlan_subifs--;
+ xd->num_subifs++;
+ else if (xd->num_subifs)
+ xd->num_subifs--;
- if (xd->dev_type != VNET_DPDK_DEV_ETH)
- return 0;
+ if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
+ goto done;
/* currently we program VLANS only for IXGBE VF and I40E VF */
if ((xd->pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->pmd != VNET_DPDK_PMD_I40EVF))
- return 0;
+ goto done;
if (t->sub.eth.flags.no_tags == 1)
- return 0;
+ goto done;
if ((t->sub.eth.flags.one_tag != 1) || (t->sub.eth.flags.exact_match != 1))
{
- xd->vlan_subifs = prev_subifs;
- return clib_error_return (0, "unsupported VLAN setup");
+ xd->num_subifs = prev_subifs;
+ err = clib_error_return (0, "unsupported VLAN setup");
+ goto done;
}
vlan_offload = rte_eth_dev_get_vlan_offload (xd->device_index);
if ((r = rte_eth_dev_set_vlan_offload (xd->device_index, vlan_offload)))
{
- xd->vlan_subifs = prev_subifs;
- return clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d",
- xd->device_index, r);
+ xd->num_subifs = prev_subifs;
+ err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d",
+ xd->device_index, r);
+ goto done;
}
rte_eth_dev_vlan_filter (xd->device_index, t->sub.eth.outer_vlan_id,
is_add)))
{
- xd->vlan_subifs = prev_subifs;
- return clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d",
- xd->device_index, r);
+ xd->num_subifs = prev_subifs;
+ err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d",
+ xd->device_index, r);
+ goto done;
}
- return 0;
+done:
+ if (xd->num_subifs)
+ xd->flags |= DPDK_DEVICE_FLAG_HAVE_SUBIF;
+ else
+ xd->flags &= ~DPDK_DEVICE_FLAG_HAVE_SUBIF;
+
+ return err;
}
/* *INDENT-OFF* */
.subif_add_del_function = dpdk_subif_add_del_function,
.rx_redirect_to_node = dpdk_set_interface_next_node,
.no_flatten_output_chains = 1,
- .name_renumber = dpdk_device_renumber,
+ .mac_addr_change_function = dpdk_set_mac_address,
};
VLIB_DEVICE_TX_FUNCTION_MULTIARCH (dpdk_device_class, dpdk_interface_tx)
Code Review / vpp.git / blobdiff