#include <vlib/pci/pci.h>
#include <vnet/ethernet/ethernet.h>
#include <vnet/devices/devices.h>
+#include <vnet/ip/ip6_packet.h>
+#include <vnet/ip/ip4_packet.h>
#include <vmxnet3/vmxnet3.h>
vmxnet3_txq_t * txq)
{
vmxnet3_tx_comp *tx_comp;
- u32 bi0;
vmxnet3_tx_comp_ring *comp_ring;
- u16 eop_idx, desc_idx;
comp_ring = &txq->tx_comp_ring;
tx_comp = &txq->tx_comp[comp_ring->next];
while ((tx_comp->flags & VMXNET3_TXCF_GEN) == comp_ring->gen)
{
- eop_idx = tx_comp->index & VMXNET3_TXC_INDEX;
- do
+ u16 eop_idx = tx_comp->index & VMXNET3_TXC_INDEX;
+ u32 bi0 = txq->tx_ring.bufs[txq->tx_ring.consume];
+
+ vlib_buffer_free_one (vm, bi0);
+ while (txq->tx_ring.consume != eop_idx)
{
- desc_idx = txq->tx_ring.consume;
- bi0 = txq->tx_ring.bufs[desc_idx];
- txq->tx_ring.bufs[desc_idx] = ~0;
- vlib_buffer_free_no_next (vm, &bi0, 1);
vmxnet3_tx_ring_advance_consume (txq);
}
- while (desc_idx != eop_idx);
+ vmxnet3_tx_ring_advance_consume (txq);
vmxnet3_tx_comp_ring_advance_next (txq);
tx_comp = &txq->tx_comp[comp_ring->next];
vmxnet3_main_t *vmxm = &vmxnet3_main;
vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, rd->dev_instance);
- u32 *buffers = vlib_frame_args (frame);
+ u32 *buffers = vlib_frame_vector_args (frame);
u32 bi0;
vlib_buffer_t *b0;
- vmxnet3_tx_desc *txd;
+ vmxnet3_tx_desc *txd = 0;
u32 desc_idx, generation, first_idx;
u16 space_left;
u16 n_left = frame->n_vectors;
vmxnet3_txq_t *txq;
- u32 thread_index = vm->thread_index;
- u16 qid = thread_index;
- u16 n_retry = 5;
+ vnet_hw_if_tx_frame_t *tf = vlib_frame_scalar_args (frame);
+ u16 qid = tf->queue_id, produce;
- txq = vec_elt_at_index (vd->txqs, qid % vd->num_tx_queues);
+ if (PREDICT_FALSE (!(vd->flags & VMXNET3_DEVICE_F_LINK_UP)))
+ {
+ vlib_buffer_free (vm, buffers, n_left);
+ vlib_error_count (vm, node->node_index, VMXNET3_TX_ERROR_LINK_DOWN,
+ n_left);
+ return (0);
+ }
- clib_spinlock_lock_if_init (&txq->lock);
+ txq = vec_elt_at_index (vd->txqs, qid);
+ if (tf->shared_queue)
+ clib_spinlock_lock (&txq->lock);
-retry:
vmxnet3_txq_release (vm, vd, txq);
- while (n_left)
+ produce = txq->tx_ring.produce;
+ while (PREDICT_TRUE (n_left))
{
+ u16 space_needed = 1, i;
+ u32 gso_size = 0;
+ u32 l4_hdr_sz;
+ vlib_buffer_t *b;
+ u32 hdr_len = 0;
+
bi0 = buffers[0];
- txd = 0;
+ b0 = vlib_get_buffer (vm, bi0);
+ b = b0;
+
+ space_left = vmxnet3_tx_ring_space_left (txq);
+ while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ u32 next_buffer = b->next_buffer;
+
+ b = vlib_get_buffer (vm, next_buffer);
+ space_needed++;
+ }
+ if (PREDICT_FALSE (space_left < space_needed))
+ {
+ vmxnet3_txq_release (vm, vd, txq);
+ space_left = vmxnet3_tx_ring_space_left (txq);
+
+ if (PREDICT_FALSE (space_left < space_needed))
+ {
+ vlib_buffer_free_one (vm, bi0);
+ vlib_error_count (vm, node->node_index,
+ VMXNET3_TX_ERROR_NO_FREE_SLOTS, 1);
+ buffers++;
+ n_left--;
+ /*
+ * Drop this packet. But we may have enough room for the next
+ * packet
+ */
+ continue;
+ }
+ }
/*
* Toggle the generation bit for SOP fragment to avoid device starts
*/
generation = txq->tx_ring.gen ^ VMXNET3_TXF_GEN;
first_idx = txq->tx_ring.produce;
- while (1)
+ for (i = 0; i < space_needed; i++)
{
b0 = vlib_get_buffer (vm, bi0);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-
- space_left = vmxnet3_tx_ring_space_left (txq);
- if (PREDICT_FALSE (space_left == 0))
- {
- break;
- }
desc_idx = txq->tx_ring.produce;
txq->tx_ring.bufs[desc_idx] = bi0;
txd = &txq->tx_desc[desc_idx];
- txd->address =
- vlib_get_buffer_data_physical_address (vm,
- bi0) + b0->current_data;
- txd->flags[0] = generation | b0->current_length;
+ txd->address = vlib_buffer_get_current_pa (vm, b0);
- generation = txq->tx_ring.gen;
- if (b0->flags & VLIB_BUFFER_NEXT_PRESENT)
+ txd->flags[0] = generation | b0->current_length;
+ txd->flags[1] = 0;
+ if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO))
{
- txd->flags[1] = 0;
- bi0 = b0->next_buffer;
+ /*
+ * We should not be getting GSO outbound traffic unless it is
+ * lro is enable
+ */
+ ASSERT (vd->gso_enable == 1);
+ gso_size = vnet_buffer2 (b0)->gso_size;
+ l4_hdr_sz = vnet_buffer2 (b0)->gso_l4_hdr_sz;
+ if (b0->flags & VNET_BUFFER_F_IS_IP6)
+ hdr_len = sizeof (ethernet_header_t) + sizeof (ip6_header_t) +
+ l4_hdr_sz;
+ else
+ hdr_len = sizeof (ethernet_header_t) + sizeof (ip4_header_t) +
+ l4_hdr_sz;
}
- else
- break;
- }
- if (PREDICT_TRUE (txd != 0))
- {
- txd->flags[1] = VMXNET3_TXF_CQ | VMXNET3_TXF_EOP;
- asm volatile ("":::"memory");
- /*
- * Now toggle back the generation bit for the first segment.
- * Device can start reading the packet
- */
- txq->tx_desc[first_idx].flags[0] ^= VMXNET3_TXF_GEN;
- vmxnet3_reg_write (vd, 0, VMXNET3_REG_TXPROD, txq->tx_ring.produce);
+ generation = txq->tx_ring.gen;
+ bi0 = b0->next_buffer;
}
-
- if (PREDICT_FALSE (space_left == 0))
+ if (PREDICT_FALSE (gso_size != 0))
{
- break;
+ txq->tx_desc[first_idx].flags[1] = hdr_len;
+ txq->tx_desc[first_idx].flags[1] |= VMXNET3_TXF_OM (VMXNET3_OM_TSO);
+ txq->tx_desc[first_idx].flags[0] |= VMXNET3_TXF_MSSCOF (gso_size);
}
+ txd->flags[1] |= VMXNET3_TXF_CQ | VMXNET3_TXF_EOP;
+ asm volatile ("":::"memory");
+ /*
+ * Now toggle back the generation bit for the first segment.
+ * Device can start reading the packet
+ */
+ txq->tx_desc[first_idx].flags[0] ^= VMXNET3_TXF_GEN;
buffers++;
n_left--;
}
- if (PREDICT_FALSE (n_left))
- {
- if (PREDICT_TRUE (n_retry--))
- goto retry;
- vlib_buffer_free (vm, buffers, n_left);
- vlib_error_count (vm, node->node_index, VMXNET3_TX_ERROR_NO_FREE_SLOTS,
- n_left);
- }
- clib_spinlock_unlock_if_init (&txq->lock);
+ if (PREDICT_TRUE (produce != txq->tx_ring.produce))
+ vmxnet3_reg_write_inline (vd, 0, txq->reg_txprod, txq->tx_ring.produce);
+
+ if (tf->shared_queue)
+ clib_spinlock_unlock (&txq->lock);
return (frame->n_vectors - n_left);
}