+ vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
+ avf_device_t *ad = avf_get_device (rd->dev_instance);
+ vnet_hw_if_tx_frame_t *tf = vlib_frame_scalar_args (frame);
+ u8 qid = tf->queue_id;
+ avf_txq_t *txq = vec_elt_at_index (ad->txqs, qid);
+ u16 next;
+ u16 mask = txq->size - 1;
+ u32 *buffers = vlib_frame_vector_args (frame);
+ u16 n_enq, n_left, n_desc, *slot;
+ u16 n_retry = 2;
+
+ if (tf->shared_queue)
+ clib_spinlock_lock (&txq->lock);
+
+ n_left = frame->n_vectors;
+
+retry:
+ next = txq->next;
+ /* release consumed bufs */
+ if (txq->n_enqueued)
+ {
+ i32 complete_slot = -1;
+ while (1)
+ {
+ u16 *slot = clib_ring_get_first (txq->rs_slots);
+
+ if (slot == 0)
+ break;
+
+ if (avf_tx_desc_get_dtyp (txq->descs + slot[0]) != 0x0F)
+ break;
+
+ complete_slot = slot[0];
+
+ clib_ring_deq (txq->rs_slots);
+ }
+
+ if (complete_slot >= 0)
+ {
+ u16 first, mask, n_free;
+ mask = txq->size - 1;
+ first = (txq->next - txq->n_enqueued) & mask;
+ n_free = (complete_slot + 1 - first) & mask;
+
+ txq->n_enqueued -= n_free;
+ vlib_buffer_free_from_ring_no_next (vm, txq->bufs, first, txq->size,
+ n_free);
+ }
+ }
+
+ n_desc = 0;
+ if (ad->flags & AVF_DEVICE_F_VA_DMA)
+ n_enq = avf_tx_prepare (vm, node, txq, buffers, n_left, &n_desc, 1);
+ else
+ n_enq = avf_tx_prepare (vm, node, txq, buffers, n_left, &n_desc, 0);
+
+ if (n_desc)
+ {
+ if (PREDICT_TRUE (next + n_desc <= txq->size))
+ {
+ /* no wrap */
+ avf_tx_copy_desc (txq->descs + next, txq->tmp_descs, n_desc);
+ vlib_buffer_copy_indices (txq->bufs + next, txq->tmp_bufs, n_desc);
+ }
+ else
+ {
+ /* wrap */
+ u32 n_not_wrap = txq->size - next;
+ avf_tx_copy_desc (txq->descs + next, txq->tmp_descs, n_not_wrap);
+ avf_tx_copy_desc (txq->descs, txq->tmp_descs + n_not_wrap,
+ n_desc - n_not_wrap);
+ vlib_buffer_copy_indices (txq->bufs + next, txq->tmp_bufs,
+ n_not_wrap);
+ vlib_buffer_copy_indices (txq->bufs, txq->tmp_bufs + n_not_wrap,
+ n_desc - n_not_wrap);
+ }
+
+ next += n_desc;
+ if ((slot = clib_ring_enq (txq->rs_slots)))
+ {
+ u16 rs_slot = slot[0] = (next - 1) & mask;
+ txq->descs[rs_slot].qword[1] |= AVF_TXD_CMD_RS;
+ }
+
+ txq->next = next & mask;
+ avf_tail_write (txq->qtx_tail, txq->next);
+ txq->n_enqueued += n_desc;
+ n_left -= n_enq;
+ }
+
+ if (n_left)
+ {
+ buffers += n_enq;
+
+ if (n_retry--)
+ goto retry;
+
+ vlib_buffer_free (vm, buffers, n_left);
+ vlib_error_count (vm, node->node_index,
+ AVF_TX_ERROR_NO_FREE_SLOTS, n_left);
+ }
+
+ if (tf->shared_queue)
+ clib_spinlock_unlock (&txq->lock);
+
+ return frame->n_vectors - n_left;