- if (PREDICT_FALSE(xd->dev_type != VNET_DPDK_DEV_VHOST_USER &&
- xd->lockp != 0))
- {
- queue_id = queue_id % xd->tx_q_used;
- while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
- /* zzzz */
- queue_id = (queue_id + 1) % xd->tx_q_used;
- }
-
- if (PREDICT_TRUE(xd->dev_type == VNET_DPDK_DEV_ETH))
- {
- if (PREDICT_TRUE(tx_head > tx_tail))
- {
- /* no wrap, transmit in one burst */
- rv = rte_eth_tx_burst(xd->device_index,
- (uint16_t) queue_id,
- &tx_vector[tx_tail],
- (uint16_t) (tx_head-tx_tail));
- }
- else
- {
- /*
- * This can only happen if there is a flowcontrol callback.
- * We need to split the transmit into two calls: one for
- * the packets up to the wrap point, and one to continue
- * at the start of the ring.
- * Transmit pkts up to the wrap point.
- */
- rv = rte_eth_tx_burst(xd->device_index,
- (uint16_t) queue_id,
- &tx_vector[tx_tail],
- (uint16_t) (DPDK_TX_RING_SIZE - tx_tail));
-
- /*
- * If we transmitted everything we wanted, then allow 1 retry
- * so we can try to transmit the rest. If we didn't transmit
- * everything, stop now.
- */
- n_retry = (rv == DPDK_TX_RING_SIZE - tx_tail) ? 1 : 0;
- }
- }
- else if (xd->dev_type == VNET_DPDK_DEV_VHOST_USER)
- {
- u32 offset = 0;
- if (xd->need_txlock) {
- queue_id = 0;
- while (__sync_lock_test_and_set (xd->lockp[queue_id], 1));
- }
-#if RTE_VERSION >= RTE_VERSION_NUM(2, 2, 0, 0)
- else {
- dpdk_device_and_queue_t * dq;
- vec_foreach (dq, dm->devices_by_cpu[vm->cpu_index])
- {
- if (xd->device_index == dq->device)
- break;
- }
- assert (dq);
- offset = dq->queue_id * VIRTIO_QNUM;
- }
+ if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_VHOST_USER) == 0 &&
+ xd->lockp != 0))
+ {
+ queue_id = queue_id % xd->tx_q_used;
+ while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
+ /* zzzz */
+ queue_id = (queue_id + 1) % xd->tx_q_used;
+ }
+
+ if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_HQOS)) /* HQoS ON */
+ {
+ if (PREDICT_TRUE (tx_head > tx_tail))
+ {
+ /* no wrap, transmit in one burst */
+ dpdk_device_hqos_per_worker_thread_t *hqos =
+ &xd->hqos_wt[vm->cpu_index];
+
+ dpdk_hqos_metadata_set (hqos,
+ &tx_vector[tx_tail], tx_head - tx_tail);
+ rv = rte_ring_sp_enqueue_burst (hqos->swq,
+ (void **) &tx_vector[tx_tail],
+ (uint16_t) (tx_head - tx_tail));
+ }
+ else
+ {
+ /*
+ * This can only happen if there is a flowcontrol callback.
+ * We need to split the transmit into two calls: one for
+ * the packets up to the wrap point, and one to continue
+ * at the start of the ring.
+ * Transmit pkts up to the wrap point.
+ */
+ dpdk_device_hqos_per_worker_thread_t *hqos =
+ &xd->hqos_wt[vm->cpu_index];
+
+ dpdk_hqos_metadata_set (hqos,
+ &tx_vector[tx_tail],
+ xd->nb_tx_desc - tx_tail);
+ rv = rte_ring_sp_enqueue_burst (hqos->swq,
+ (void **) &tx_vector[tx_tail],
+ (uint16_t) (xd->nb_tx_desc -
+ tx_tail));
+ /*
+ * If we transmitted everything we wanted, then allow 1 retry
+ * so we can try to transmit the rest. If we didn't transmit
+ * everything, stop now.
+ */
+ n_retry = (rv == xd->nb_tx_desc - tx_tail) ? 1 : 0;
+ }
+ }
+ else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
+ {
+ if (PREDICT_TRUE (tx_head > tx_tail))
+ {
+ /* no wrap, transmit in one burst */
+ rv = rte_eth_tx_burst (xd->device_index,
+ (uint16_t) queue_id,
+ &tx_vector[tx_tail],
+ (uint16_t) (tx_head - tx_tail));
+ }
+ else
+ {
+ /*
+ * This can only happen if there is a flowcontrol callback.
+ * We need to split the transmit into two calls: one for
+ * the packets up to the wrap point, and one to continue
+ * at the start of the ring.
+ * Transmit pkts up to the wrap point.
+ */
+ rv = rte_eth_tx_burst (xd->device_index,
+ (uint16_t) queue_id,
+ &tx_vector[tx_tail],
+ (uint16_t) (xd->nb_tx_desc - tx_tail));
+
+ /*
+ * If we transmitted everything we wanted, then allow 1 retry
+ * so we can try to transmit the rest. If we didn't transmit
+ * everything, stop now.
+ */
+ n_retry = (rv == xd->nb_tx_desc - tx_tail) ? 1 : 0;
+ }
+ }
+#if DPDK_VHOST_USER
+ else if (xd->flags & DPDK_DEVICE_FLAG_VHOST_USER)
+ {
+ u32 offset = 0;
+ if (xd->need_txlock)
+ {
+ queue_id = 0;
+ while (__sync_lock_test_and_set (xd->lockp[queue_id], 1));
+ }
+ else
+ {
+ dpdk_device_and_queue_t *dq;
+ vec_foreach (dq, dm->devices_by_cpu[vm->cpu_index])
+ {
+ if (xd->device_index == dq->device)
+ break;
+ }
+ assert (dq);
+ offset = dq->queue_id * VIRTIO_QNUM;
+ }
+ if (PREDICT_TRUE (tx_head > tx_tail))
+ {
+ int i;
+ u32 bytes = 0;
+ struct rte_mbuf **pkts = &tx_vector[tx_tail];
+ for (i = 0; i < (tx_head - tx_tail); i++)
+ {
+ struct rte_mbuf *buff = pkts[i];
+ bytes += rte_pktmbuf_data_len (buff);
+ }
+
+ /* no wrap, transmit in one burst */
+ rv =
+ rte_vhost_enqueue_burst (&xd->vu_vhost_dev,
+ offset + VIRTIO_RXQ,
+ &tx_vector[tx_tail],
+ (uint16_t) (tx_head - tx_tail));
+ if (PREDICT_TRUE (rv > 0))
+ {
+ dpdk_vu_vring *vring =
+ &(xd->vu_intf->vrings[offset + VIRTIO_TXQ]);
+ vring->packets += rv;
+ vring->bytes += bytes;
+
+ if (dpdk_vhost_user_want_interrupt
+ (xd, offset + VIRTIO_RXQ))
+ {
+ vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]);
+ vring->n_since_last_int += rv;
+
+ f64 now = vlib_time_now (vm);
+ if (vring->int_deadline < now ||
+ vring->n_since_last_int >
+ dm->conf->vhost_coalesce_frames)
+ dpdk_vhost_user_send_interrupt (vm, xd,
+ offset + VIRTIO_RXQ);
+ }
+
+ int c = rv;
+ while (c--)
+ rte_pktmbuf_free (tx_vector[tx_tail + c]);
+ }
+ }
+ else
+ {
+ /*
+ * If we transmitted everything we wanted, then allow 1 retry
+ * so we can try to transmit the rest. If we didn't transmit
+ * everything, stop now.
+ */
+ int i;
+ u32 bytes = 0;
+ struct rte_mbuf **pkts = &tx_vector[tx_tail];
+ for (i = 0; i < (xd->nb_tx_desc - tx_tail); i++)
+ {
+ struct rte_mbuf *buff = pkts[i];
+ bytes += rte_pktmbuf_data_len (buff);
+ }
+ rv =
+ rte_vhost_enqueue_burst (&xd->vu_vhost_dev,
+ offset + VIRTIO_RXQ,
+ &tx_vector[tx_tail],
+ (uint16_t) (xd->nb_tx_desc -
+ tx_tail));
+
+ if (PREDICT_TRUE (rv > 0))
+ {
+ dpdk_vu_vring *vring =
+ &(xd->vu_intf->vrings[offset + VIRTIO_TXQ]);
+ vring->packets += rv;
+ vring->bytes += bytes;
+
+ if (dpdk_vhost_user_want_interrupt
+ (xd, offset + VIRTIO_RXQ))
+ {
+ vring = &(xd->vu_intf->vrings[offset + VIRTIO_RXQ]);
+ vring->n_since_last_int += rv;
+
+ f64 now = vlib_time_now (vm);
+ if (vring->int_deadline < now ||
+ vring->n_since_last_int >
+ dm->conf->vhost_coalesce_frames)
+ dpdk_vhost_user_send_interrupt (vm, xd,
+ offset + VIRTIO_RXQ);
+ }
+
+ int c = rv;
+ while (c--)
+ rte_pktmbuf_free (tx_vector[tx_tail + c]);
+ }
+
+ n_retry = (rv == xd->nb_tx_desc - tx_tail) ? 1 : 0;
+ }
+
+ if (xd->need_txlock)
+ *xd->lockp[queue_id] = 0;
+ }