X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Favf%2Foutput.c;h=6c43885569e6c6fdbb856bda50940f44a0e5f779;hb=f7ab7b2d9bc0f42c1e766d22d49dd0dc4f28abb6;hp=d5e71d528abfedbcbd7f58521f35502d767b2def;hpb=c22f87b8763c844e09177e30bbb92830cd928812;p=vpp.git

diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c
index d5e71d528ab..6c43885569e 100644
--- a/src/plugins/avf/output.c
+++ b/src/plugins/avf/output.c
@@ -18,162 +18,326 @@
 #include <vlib/vlib.h>
 #include <vlib/unix/unix.h>
 #include <vlib/pci/pci.h>
+#include <vppinfra/ring.h>
 #include <vnet/ethernet/ethernet.h>
 #include <vnet/devices/devices.h>
 
 #include <avf/avf.h>
 
-#define AVF_TXQ_DESC_CMD(x)             (1 << (x + 4))
-#define AVF_TXQ_DESC_CMD_EOP		AVF_TXQ_DESC_CMD(0)
-#define AVF_TXQ_DESC_CMD_RS		AVF_TXQ_DESC_CMD(1)
-#define AVF_TXQ_DESC_CMD_RSV		AVF_TXQ_DESC_CMD(2)
-
 static_always_inline u8
 avf_tx_desc_get_dtyp (avf_tx_desc_t * d)
 {
   return d->qword[1] & 0x0f;
 }
 
-uword
-CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm,
-				      vlib_node_runtime_t * node,
-				      vlib_frame_t * frame)
+static_always_inline u16
+avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
+		u32 * buffers, u32 n_packets, int use_va_dma)
+{
+  u16 next = txq->next;
+  u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV;
+  u16 n_desc = 0;
+  u16 *slot, n_desc_left, n_packets_left = n_packets;
+  u16 mask = txq->size - 1;
+  vlib_buffer_t *b[4];
+  avf_tx_desc_t *d = txq->descs + next;
+  u16 n_desc_needed;
+  vlib_buffer_t *b0;
+
+  /* avoid ring wrap */
+  n_desc_left = txq->size - clib_max (txq->next, txq->n_enqueued + 8);
+
+  if (n_desc_left == 0)
+    return 0;
+
+  /* Fast path, no ring wrap */
+  while (n_packets_left && n_desc_left)
+    {
+      u32 or_flags;
+      if (n_packets_left < 8 || n_desc_left < 4)
+	goto one_by_one;
+
+      vlib_prefetch_buffer_with_index (vm, buffers[4], LOAD);
+      vlib_prefetch_buffer_with_index (vm, buffers[5], LOAD);
+      vlib_prefetch_buffer_with_index (vm, buffers[6], LOAD);
+      vlib_prefetch_buffer_with_index (vm, buffers[7], LOAD);
+
+      b[0] = vlib_get_buffer (vm, buffers[0]);
+      b[1] = vlib_get_buffer (vm, buffers[1]);
+      b[2] = vlib_get_buffer (vm, buffers[2]);
+      b[3] = vlib_get_buffer (vm, buffers[3]);
+
+      or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
+
+      if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
+	goto one_by_one;
+
+      vlib_buffer_copy_indices (txq->bufs + next, buffers, 4);
+
+      if (use_va_dma)
+	{
+	  d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
+	  d[1].qword[0] = vlib_buffer_get_current_va (b[1]);
+	  d[2].qword[0] = vlib_buffer_get_current_va (b[2]);
+	  d[3].qword[0] = vlib_buffer_get_current_va (b[3]);
+	}
+      else
+	{
+	  d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
+	  d[1].qword[0] = vlib_buffer_get_current_pa (vm, b[1]);
+	  d[2].qword[0] = vlib_buffer_get_current_pa (vm, b[2]);
+	  d[3].qword[0] = vlib_buffer_get_current_pa (vm, b[3]);
+	}
+
+      d[0].qword[1] = ((u64) b[0]->current_length) << 34 | bits;
+      d[1].qword[1] = ((u64) b[1]->current_length) << 34 | bits;
+      d[2].qword[1] = ((u64) b[2]->current_length) << 34 | bits;
+      d[3].qword[1] = ((u64) b[3]->current_length) << 34 | bits;
+
+      next += 4;
+      n_desc += 4;
+      buffers += 4;
+      n_packets_left -= 4;
+      n_desc_left -= 4;
+      d += 4;
+      continue;
+
+    one_by_one:
+      txq->bufs[next] = buffers[0];
+      b[0] = vlib_get_buffer (vm, buffers[0]);
+
+      /* Deal with chain buffer if present */
+      if (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
+	{
+	  n_desc_needed = 1;
+	  b0 = b[0];
+
+	  /* Wish there were a buffer count for chain buffer */
+	  while (b0->flags & VLIB_BUFFER_NEXT_PRESENT)
+	    {
+	      b0 = vlib_get_buffer (vm, b0->next_buffer);
+	      n_desc_needed++;
+	    }
+
+	  /* spec says data descriptor is limited to 8 segments */
+	  if (PREDICT_FALSE (n_desc_needed > 8))
+	    {
+	      vlib_buffer_free_one (vm, buffers[0]);
+	      vlib_error_count (vm, node->node_index,
+				AVF_TX_ERROR_SEGMENT_SIZE_EXCEEDED, 1);
+	      n_packets_left -= 1;
+	      buffers += 1;
+	      continue;
+	    }
+
+	  if (PREDICT_FALSE (n_desc_left < n_desc_needed))
+	    /*
+	     * Slow path may be able to to deal with this since it can handle
+	     * ring wrap
+	     */
+	    break;
+
+	  while (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
+	    {
+	      if (use_va_dma)
+		d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
+	      else
+		d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
+
+	      d[0].qword[1] = (((u64) b[0]->current_length) << 34) |
+		AVF_TXD_CMD_RSV;
+
+	      next += 1;
+	      n_desc += 1;
+	      n_desc_left -= 1;
+	      d += 1;
+
+	      txq->bufs[next] = b[0]->next_buffer;
+	      b[0] = vlib_get_buffer (vm, b[0]->next_buffer);
+	    }
+	}
+
+      if (use_va_dma)
+	d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
+      else
+	d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
+
+      d[0].qword[1] = (((u64) b[0]->current_length) << 34) | bits;
+
+      next += 1;
+      n_desc += 1;
+      buffers += 1;
+      n_packets_left -= 1;
+      n_desc_left -= 1;
+      d += 1;
+    }
+
+  /* Slow path to support ring wrap */
+  if (PREDICT_FALSE (n_packets_left))
+    {
+      txq->n_enqueued += n_desc;
+
+      n_desc = 0;
+      d = txq->descs + (next & mask);
+
+      /* +8 to be consistent with fast path */
+      n_desc_left = txq->size - (txq->n_enqueued + 8);
+
+      while (n_packets_left && n_desc_left)
+	{
+	  txq->bufs[next & mask] = buffers[0];
+	  b[0] = vlib_get_buffer (vm, buffers[0]);
+
+	  /* Deal with chain buffer if present */
+	  if (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
+	    {
+	      n_desc_needed = 1;
+	      b0 = b[0];
+
+	      while (b0->flags & VLIB_BUFFER_NEXT_PRESENT)
+		{
+		  b0 = vlib_get_buffer (vm, b0->next_buffer);
+		  n_desc_needed++;
+		}
+
+	      /* Spec says data descriptor is limited to 8 segments */
+	      if (PREDICT_FALSE (n_desc_needed > 8))
+		{
+		  vlib_buffer_free_one (vm, buffers[0]);
+		  vlib_error_count (vm, node->node_index,
+				    AVF_TX_ERROR_SEGMENT_SIZE_EXCEEDED, 1);
+		  n_packets_left -= 1;
+		  buffers += 1;
+		  continue;
+		}
+
+	      if (PREDICT_FALSE (n_desc_left < n_desc_needed))
+		break;
+
+	      while (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
+		{
+		  if (use_va_dma)
+		    d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
+		  else
+		    d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
+
+		  d[0].qword[1] = (((u64) b[0]->current_length) << 34) |
+		    AVF_TXD_CMD_RSV;
+
+		  next += 1;
+		  n_desc += 1;
+		  n_desc_left -= 1;
+		  d = txq->descs + (next & mask);
+
+		  txq->bufs[next & mask] = b[0]->next_buffer;
+		  b[0] = vlib_get_buffer (vm, b[0]->next_buffer);
+		}
+	    }
+
+	  if (use_va_dma)
+	    d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
+	  else
+	    d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
+
+	  d[0].qword[1] = (((u64) b[0]->current_length) << 34) | bits;
+
+	  next += 1;
+	  n_desc += 1;
+	  buffers += 1;
+	  n_packets_left -= 1;
+	  n_desc_left -= 1;
+	  d = txq->descs + (next & mask);
+	}
+    }
+
+  if ((slot = clib_ring_enq (txq->rs_slots)))
+    {
+      u16 rs_slot = slot[0] = (next - 1) & mask;
+      d = txq->descs + rs_slot;
+      d[0].qword[1] |= AVF_TXD_CMD_RS;
+    }
+
+  CLIB_MEMORY_BARRIER ();
+  *(txq->qtx_tail) = txq->next = next & mask;
+  txq->n_enqueued += n_desc;
+  return n_packets - n_packets_left;
+}
+
+VNET_DEVICE_CLASS_TX_FN (avf_device_class) (vlib_main_t * vm,
+					    vlib_node_runtime_t * node,
+					    vlib_frame_t * frame)
 {
   avf_main_t *am = &avf_main;
   vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
   avf_device_t *ad = pool_elt_at_index (am->devices, rd->dev_instance);
-  u32 thread_index = vlib_get_thread_index ();
+  u32 thread_index = vm->thread_index;
   u8 qid = thread_index;
   avf_txq_t *txq = vec_elt_at_index (ad->txqs, qid % ad->num_queue_pairs);
-  avf_tx_desc_t *d0, *d1, *d2, *d3;
-  u32 *buffers = vlib_frame_args (frame);
-  u32 bi0, bi1, bi2, bi3;
-  u16 n_left = frame->n_vectors;
-  vlib_buffer_t *b0, *b1, *b2, *b3;
-  u16 mask = txq->size - 1;
-  u64 bits = (AVF_TXQ_DESC_CMD_EOP | AVF_TXQ_DESC_CMD_RS |
-	      AVF_TXQ_DESC_CMD_RSV);
+  u32 *buffers = vlib_frame_vector_args (frame);
+  u16 n_enq, n_left;
+  u16 n_retry = 2;
 
   clib_spinlock_lock_if_init (&txq->lock);
 
-  /* release cosumed bufs */
-  if (txq->n_bufs)
+  n_left = frame->n_vectors;
+
+retry:
+  /* release consumed bufs */
+  if (txq->n_enqueued)
     {
-      u16 first, slot, n_free = 0;
-      first = slot = (txq->next - txq->n_bufs) & mask;
-      d0 = txq->descs + slot;
-      while (n_free < txq->n_bufs && avf_tx_desc_get_dtyp (d0) == 0x0F)
+      i32 complete_slot = -1;
+      while (1)
 	{
-	  n_free++;
-	  slot = (slot + 1) & mask;
-	  d0 = txq->descs + slot;
+	  u16 *slot = clib_ring_get_first (txq->rs_slots);
+
+	  if (slot == 0)
+	    break;
+
+	  if (avf_tx_desc_get_dtyp (txq->descs + slot[0]) != 0x0F)
+	    break;
+
+	  complete_slot = slot[0];
+
+	  clib_ring_deq (txq->rs_slots);
 	}
 
-      if (n_free)
+      if (complete_slot >= 0)
 	{
-	  txq->n_bufs -= n_free;;
-	  vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size,
-				      n_free);
+	  u16 first, mask, n_free;
+	  mask = txq->size - 1;
+	  first = (txq->next - txq->n_enqueued) & mask;
+	  n_free = (complete_slot + 1 - first) & mask;
+
+	  txq->n_enqueued -= n_free;
+	  vlib_buffer_free_from_ring_no_next (vm, txq->bufs, first, txq->size,
+					      n_free);
 	}
     }
 
-  while (n_left >= 7)
-    {
-      u16 slot0, slot1, slot2, slot3;
+  if (ad->flags & AVF_DEVICE_F_VA_DMA)
+    n_enq = avf_tx_enqueue (vm, node, txq, buffers, n_left, 1);
+  else
+    n_enq = avf_tx_enqueue (vm, node, txq, buffers, n_left, 0);
 
-      vlib_prefetch_buffer_with_index (vm, buffers[4], LOAD);
-      vlib_prefetch_buffer_with_index (vm, buffers[5], LOAD);
-      vlib_prefetch_buffer_with_index (vm, buffers[6], LOAD);
-      vlib_prefetch_buffer_with_index (vm, buffers[7], LOAD);
+  n_left -= n_enq;
 
-      slot0 = txq->next;
-      slot1 = (txq->next + 1) & mask;
-      slot2 = (txq->next + 2) & mask;
-      slot3 = (txq->next + 3) & mask;
-
-      d0 = txq->descs + slot0;
-      d1 = txq->descs + slot1;
-      d2 = txq->descs + slot2;
-      d3 = txq->descs + slot3;
-
-      bi0 = buffers[0];
-      bi1 = buffers[1];
-      bi2 = buffers[2];
-      bi3 = buffers[3];
-
-      txq->bufs[slot0] = bi0;
-      txq->bufs[slot1] = bi1;
-      txq->bufs[slot2] = bi2;
-      txq->bufs[slot3] = bi3;
-      b0 = vlib_get_buffer (vm, bi0);
-      b1 = vlib_get_buffer (vm, bi1);
-      b2 = vlib_get_buffer (vm, bi2);
-      b3 = vlib_get_buffer (vm, bi3);
-
-#if 0
-      d->qword[0] = vlib_get_buffer_data_physical_address (vm, bi0) +
-	b0->current_data;
-#else
-      d0->qword[0] = pointer_to_uword (b0->data) + b0->current_data;
-      d1->qword[0] = pointer_to_uword (b1->data) + b1->current_data;
-      d2->qword[0] = pointer_to_uword (b2->data) + b2->current_data;
-      d3->qword[0] = pointer_to_uword (b3->data) + b3->current_data;
-
-#endif
-      d0->qword[1] = ((u64) b0->current_length) << 34 | bits;
-      d1->qword[1] = ((u64) b1->current_length) << 34 | bits;
-      d2->qword[1] = ((u64) b2->current_length) << 34 | bits;
-      d3->qword[1] = ((u64) b3->current_length) << 34 | bits;
-
-      txq->next = (txq->next + 4) & mask;
-      txq->n_bufs += 4;
-      buffers += 4;
-      n_left -= 4;
-    }
-
-  while (n_left)
+  if (n_left)
     {
-      d0 = txq->descs + txq->next;
-      bi0 = buffers[0];
-      txq->bufs[txq->next] = bi0;
-      b0 = vlib_get_buffer (vm, bi0);
-
-#if 0
-      d->qword[0] = vlib_get_buffer_data_physical_address (vm, bi0) +
-	b0->current_data;
-#else
-      d0->qword[0] = pointer_to_uword (b0->data) + b0->current_data;
-#endif
-      d0->qword[1] = (((u64) b0->current_length) << 34) | bits;
-
-      txq->next = (txq->next + 1) & mask;
-      txq->n_bufs++;
-      buffers++;
-      n_left--;
+      buffers += n_enq;
+
+      if (n_retry--)
+	goto retry;
+
+      vlib_buffer_free (vm, buffers, n_left);
+      vlib_error_count (vm, node->node_index,
+			AVF_TX_ERROR_NO_FREE_SLOTS, n_left);
     }
-  CLIB_MEMORY_BARRIER ();
-  *(txq->qtx_tail) = txq->next;
 
   clib_spinlock_unlock_if_init (&txq->lock);
 
   return frame->n_vectors - n_left;
 }
 
-#ifndef CLIB_MARCH_VARIANT
-#if __x86_64__
-vlib_node_function_t __clib_weak avf_interface_tx_avx512;
-vlib_node_function_t __clib_weak avf_interface_tx_avx2;
-static void __clib_constructor
-avf_interface_tx_multiarch_select (void)
-{
-  if (avf_interface_tx_avx512 && clib_cpu_supports_avx512f ())
-    avf_device_class.tx_function = avf_interface_tx_avx512;
-  else if (avf_interface_tx_avx2 && clib_cpu_supports_avx2 ())
-    avf_device_class.tx_function = avf_interface_tx_avx2;
-}
-#endif
-#endif
-
 /*
  * fd.io coding-style-patch-verification: ON
  *