X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fbuffer.c;h=80c6442f463aedfed6094627f4ccba55b03e1c7b;hb=04f3db3847d242857b9d9d858bcdca538a1be7d7;hp=e09d80194ede167498218a4215e5778787d1f5db;hpb=2f9b0c05fca7ca829ea438da1d87e2bf93969500;p=vpp.git

diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
index e09d80194ed..80c6442f463 100644
--- a/src/plugins/dpdk/buffer.c
+++ b/src/plugins/dpdk/buffer.c
@@ -79,6 +79,8 @@
 STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
 	       "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
 
+static struct rte_mbuf ***mbuf_pending_free_list = 0;
+
 static_always_inline void
 dpdk_rte_pktmbuf_free (vlib_main_t * vm, vlib_buffer_t * b)
 {
@@ -86,6 +88,8 @@ dpdk_rte_pktmbuf_free (vlib_main_t * vm, vlib_buffer_t * b)
   struct rte_mbuf *mb;
   u32 next, flags;
   mb = rte_mbuf_from_vlib_buffer (hb);
+  static struct rte_mempool *last_pool = 0;
+  static u8 last_buffer_pool_index;
 
 next:
   flags = b->flags;
@@ -98,7 +102,20 @@ next:
       b->n_add_refs = 0;
     }
 
-  rte_pktmbuf_free_seg (mb);
+  mb = rte_pktmbuf_prefree_seg (mb);
+  if (mb)
+    {
+      if (mb->pool != last_pool)
+	{
+	  last_pool = mb->pool;
+	  dpdk_mempool_private_t *privp = rte_mempool_get_priv (last_pool);
+	  last_buffer_pool_index = privp->buffer_pool_index;
+	  vec_validate_aligned (mbuf_pending_free_list,
+				last_buffer_pool_index,
+				CLIB_CACHE_LINE_BYTES);
+	}
+      vec_add1 (mbuf_pending_free_list[last_buffer_pool_index], mb);
+    }
 
   if (flags & VLIB_BUFFER_NEXT_PRESENT)
     {
@@ -107,6 +124,7 @@ next:
     }
 }
 
+#ifndef CLIB_MULTIARCH_VARIANT
 static void
 del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
 {
@@ -159,6 +177,7 @@ dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
       pool_put (bm->buffer_free_list_pool, f);
     }
 }
+#endif
 
 /* Make sure free list has at least given number of free buffers. */
 static uword
@@ -171,7 +190,9 @@ fill_free_list (vlib_main_t * vm,
   u32 bi0, bi1, bi2, bi3;
   unsigned socket_id = rte_socket_id ();
   struct rte_mempool *rmp = dm->pktmbuf_pools[socket_id];
+  dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp);
   struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
+  vlib_buffer_t bt;
 
   /* Too early? */
   if (PREDICT_FALSE (rmp == 0))
@@ -188,14 +209,20 @@ fill_free_list (vlib_main_t * vm,
   /* Always allocate new buffers in reasonably large sized chunks. */
   n = clib_max (n, fl->min_n_buffers_each_physmem_alloc);
 
-  vec_validate (vm->mbuf_alloc_list, n - 1);
+  vec_validate_aligned (vm->mbuf_alloc_list, n - 1, CLIB_CACHE_LINE_BYTES);
 
   if (rte_mempool_get_bulk (rmp, vm->mbuf_alloc_list, n) < 0)
     return 0;
 
+  memset (&bt, 0, sizeof (vlib_buffer_t));
+  vlib_buffer_init_for_free_list (&bt, fl);
+  bt.buffer_pool_index = privp->buffer_pool_index;
+
   _vec_len (vm->mbuf_alloc_list) = n;
 
   i = 0;
+  int f = vec_len (fl->buffers);
+  vec_resize_aligned (fl->buffers, n, CLIB_CACHE_LINE_BYTES);
 
   while (i < (n - 7))
     {
@@ -223,15 +250,12 @@ fill_free_list (vlib_main_t * vm,
       bi2 = vlib_get_buffer_index (vm, b2);
       bi3 = vlib_get_buffer_index (vm, b3);
 
-      vec_add1_aligned (fl->buffers, bi0, CLIB_CACHE_LINE_BYTES);
-      vec_add1_aligned (fl->buffers, bi1, CLIB_CACHE_LINE_BYTES);
-      vec_add1_aligned (fl->buffers, bi2, CLIB_CACHE_LINE_BYTES);
-      vec_add1_aligned (fl->buffers, bi3, CLIB_CACHE_LINE_BYTES);
+      fl->buffers[f++] = bi0;
+      fl->buffers[f++] = bi1;
+      fl->buffers[f++] = bi2;
+      fl->buffers[f++] = bi3;
 
-      vlib_buffer_init_for_free_list (b0, fl);
-      vlib_buffer_init_for_free_list (b1, fl);
-      vlib_buffer_init_for_free_list (b2, fl);
-      vlib_buffer_init_for_free_list (b3, fl);
+      clib_memcpy64_x4 (b0, b1, b2, b3, &bt);
 
       if (fl->buffer_init_function)
 	{
@@ -250,9 +274,8 @@ fill_free_list (vlib_main_t * vm,
       b0 = vlib_buffer_from_rte_mbuf (mb0);
       bi0 = vlib_get_buffer_index (vm, b0);
 
-      vec_add1_aligned (fl->buffers, bi0, CLIB_CACHE_LINE_BYTES);
-
-      vlib_buffer_init_for_free_list (b0, fl);
+      fl->buffers[f++] = bi0;
+      clib_memcpy (b0, &bt, sizeof (vlib_buffer_t));
 
       if (fl->buffer_init_function)
 	fl->buffer_init_function (vm, fl, &bi0, 1);
@@ -293,7 +316,8 @@ alloc_from_free_list (vlib_main_t * vm,
    Returns number actually allocated which will be either zero or
    number requested. */
 u32
-dpdk_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+CLIB_MULTIARCH_FN (dpdk_buffer_alloc) (vlib_main_t * vm, u32 * buffers,
+				       u32 n_buffers)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
 
@@ -306,9 +330,10 @@ dpdk_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
 
 
 u32
-dpdk_buffer_alloc_from_free_list (vlib_main_t * vm,
-				  u32 * buffers,
-				  u32 n_buffers, u32 free_list_index)
+CLIB_MULTIARCH_FN (dpdk_buffer_alloc_from_free_list) (vlib_main_t * vm,
+						      u32 * buffers,
+						      u32 n_buffers,
+						      u32 free_list_index)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
   vlib_buffer_free_list_t *f;
@@ -316,14 +341,56 @@ dpdk_buffer_alloc_from_free_list (vlib_main_t * vm,
   return alloc_from_free_list (vm, f, buffers, n_buffers);
 }
 
+static_always_inline void
+dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi)
+{
+  vlib_buffer_t *b;
+  struct rte_mbuf *mb;
+  b = vlib_get_buffer (vm, bi);
+  mb = rte_mbuf_from_vlib_buffer (b);
+  CLIB_PREFETCH (mb, CLIB_CACHE_LINE_BYTES, STORE);
+  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+}
+
+static_always_inline void
+recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi,
+		 vlib_buffer_t * b)
+{
+  vlib_buffer_free_list_t *fl;
+  u32 fi;
+  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
+
+  /* The only current use of this callback: multicast recycle */
+  if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
+    {
+      int j;
+
+      vlib_buffer_add_to_free_list (vm, fl, bi,
+				    (b->flags & VLIB_BUFFER_RECYCLE) == 0);
+
+      for (j = 0; j < vec_len (bm->announce_list); j++)
+	{
+	  if (fl == bm->announce_list[j])
+	    goto already_announced;
+	}
+      vec_add1 (bm->announce_list, fl);
+    already_announced:
+      ;
+    }
+  else
+    {
+      if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
+	dpdk_rte_pktmbuf_free (vm, b);
+    }
+}
+
 static_always_inline void
 vlib_buffer_free_inline (vlib_main_t * vm,
 			 u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
-  vlib_buffer_free_list_t *fl;
-  u32 fi;
-  int i;
+  vlib_buffer_t *b0, *b1, *b2, *b3;
+  int i = 0;
   u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
 	     u32 follow_buffer_next);
 
@@ -335,36 +402,36 @@ vlib_buffer_free_inline (vlib_main_t * vm,
   if (!n_buffers)
     return;
 
-  for (i = 0; i < n_buffers; i++)
+  while (i + 7 < n_buffers)
     {
-      vlib_buffer_t *b;
+      dpdk_prefetch_buffer_by_index (vm, buffers[i + 4]);
+      dpdk_prefetch_buffer_by_index (vm, buffers[i + 5]);
+      dpdk_prefetch_buffer_by_index (vm, buffers[i + 6]);
+      dpdk_prefetch_buffer_by_index (vm, buffers[i + 7]);
+
+      b0 = vlib_get_buffer (vm, buffers[i]);
+      b1 = vlib_get_buffer (vm, buffers[i + 1]);
+      b2 = vlib_get_buffer (vm, buffers[i + 2]);
+      b3 = vlib_get_buffer (vm, buffers[i + 3]);
+
+      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
+      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
+      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2);
+      VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3);
+
+      recycle_or_free (vm, bm, buffers[i], b0);
+      recycle_or_free (vm, bm, buffers[i + 1], b1);
+      recycle_or_free (vm, bm, buffers[i + 2], b2);
+      recycle_or_free (vm, bm, buffers[i + 3], b3);
 
-      b = vlib_get_buffer (vm, buffers[i]);
+      i += 4;
+    }
+  while (i < n_buffers)
+    {
+      b0 = vlib_get_buffer (vm, buffers[i]);
       VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b);
-      fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
-
-      /* The only current use of this callback: multicast recycle */
-      if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
-	{
-	  int j;
-
-	  vlib_buffer_add_to_free_list
-	    (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0);
-
-	  for (j = 0; j < vec_len (bm->announce_list); j++)
-	    {
-	      if (fl == bm->announce_list[j])
-		goto already_announced;
-	    }
-	  vec_add1 (bm->announce_list, fl);
-	already_announced:
-	  ;
-	}
-      else
-	{
-	  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
-	    dpdk_rte_pktmbuf_free (vm, b);
-	}
+      recycle_or_free (vm, bm, buffers[i], b0);
+      i++;
     }
   if (vec_len (bm->announce_list))
     {
@@ -376,22 +443,36 @@ vlib_buffer_free_inline (vlib_main_t * vm,
 	}
       _vec_len (bm->announce_list) = 0;
     }
+
+  vec_foreach_index (i, mbuf_pending_free_list)
+  {
+    int len = vec_len (mbuf_pending_free_list[i]);
+    if (len)
+      {
+	rte_mempool_put_bulk (mbuf_pending_free_list[i][len - 1]->pool,
+			      (void *) mbuf_pending_free_list[i], len);
+	vec_reset_length (mbuf_pending_free_list[i]);
+      }
+  }
 }
 
-static void
-dpdk_buffer_free (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+void
+CLIB_MULTIARCH_FN (dpdk_buffer_free) (vlib_main_t * vm, u32 * buffers,
+				      u32 n_buffers)
 {
   vlib_buffer_free_inline (vm, buffers, n_buffers,	/* follow_buffer_next */
 			   1);
 }
 
-static void
-dpdk_buffer_free_no_next (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+void
+CLIB_MULTIARCH_FN (dpdk_buffer_free_no_next) (vlib_main_t * vm, u32 * buffers,
+					      u32 n_buffers)
 {
   vlib_buffer_free_inline (vm, buffers, n_buffers,	/* follow_buffer_next */
 			   0);
 }
 
+#ifndef CLIB_MULTIARCH_VARIANT
 static void
 dpdk_packet_template_init (vlib_main_t * vm,
 			   void *vt,
@@ -415,7 +496,13 @@ dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
 {
   dpdk_main_t *dm = &dpdk_main;
   struct rte_mempool *rmp;
-  int i;
+  dpdk_mempool_private_t priv;
+  vlib_physmem_region_t *pr;
+  vlib_physmem_region_index_t pri;
+  u8 *pool_name;
+  unsigned elt_size;
+  u32 size, obj_size;
+  i32 i, ret;
 
   vec_validate_aligned (dm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES);
 
@@ -423,29 +510,64 @@ dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
   if (dm->pktmbuf_pools[socket_id])
     return 0;
 
-  u8 *pool_name = format (0, "mbuf_pool_socket%u%c", socket_id, 0);
+  pool_name = format (0, "dpdk_mbuf_pool_socket%u%c", socket_id, 0);
+
+  elt_size = sizeof (struct rte_mbuf) +
+    VLIB_BUFFER_HDR_SIZE /* priv size */  +
+    VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE;	/*data room size */
 
-  rmp = rte_pktmbuf_pool_create ((char *) pool_name,	/* pool name */
-				 num_mbufs,	/* number of mbufs */
-				 512,	/* cache size */
-				 VLIB_BUFFER_HDR_SIZE,	/* priv size */
-				 VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE,	/* dataroom size */
-				 socket_id);	/* cpu socket */
+  obj_size = rte_mempool_calc_obj_size (elt_size, 0, 0);
+  size = rte_mempool_xmem_size (num_mbufs, obj_size, 21);
 
+  clib_error_t *error = 0;
+  error = vlib_physmem_region_alloc (vm, (char *) pool_name, size, socket_id,
+				     0, &pri);
+  if (error)
+    clib_error_report (error);
+
+  pr = vlib_physmem_get_region (vm, pri);
+
+  priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE +
+    VLIB_BUFFER_DATA_SIZE;
+  priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE;
+
+#if 0
+  /* Check that pg_shift parameter is valid. */
+  if (pg_shift > MEMPOOL_PG_SHIFT_MAX)
+    {
+      rte_errno = EINVAL;
+      return NULL;
+    }
+#endif
+  rmp = rte_mempool_create_empty ((char *) pool_name,	/* pool name */
+				  num_mbufs,	/* number of mbufs */
+				  elt_size, 512,	/* cache size */
+				  sizeof (dpdk_mempool_private_t),	/* private data size */
+				  socket_id, 0);	/* flags */
   if (rmp)
     {
-      {
-	struct rte_mempool_memhdr *memhdr;
+      rte_mempool_set_ops_byname (rmp, RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL);
 
-	STAILQ_FOREACH (memhdr, &rmp->mem_list, next)
-	  vlib_buffer_add_mem_range (vm, (uword) memhdr->addr, memhdr->len);
-      }
-      if (rmp)
+      /* call the mempool priv initializer */
+      rte_pktmbuf_pool_init (rmp, &priv);
+
+      ret = rte_mempool_populate_phys_tab (rmp, pr->mem, pr->page_table,
+					   pr->n_pages, pr->log2_page_size,
+					   NULL, NULL);
+      if (ret == (i32) rmp->size)
 	{
+	  /* call the object initializers */
+	  rte_mempool_obj_iter (rmp, rte_pktmbuf_init, 0);
+
+	  dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp);
+	  privp->buffer_pool_index = vlib_buffer_add_physmem_region (vm, pri);
+
 	  dm->pktmbuf_pools[socket_id] = rmp;
-	  vec_free (pool_name);
+
 	  return 0;
 	}
+
+      rte_mempool_free (rmp);
     }
 
   vec_free (pool_name);
@@ -455,10 +577,9 @@ dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
     {
       if (dm->pktmbuf_pools[i])
 	{
-	  clib_warning
-	    ("WARNING: Failed to allocate mempool for CPU socket %u. "
-	     "Threads running on socket %u will use socket %u mempool.",
-	     socket_id, socket_id, i);
+	  clib_warning ("WARNING: Failed to allocate mempool for CPU socket "
+			"%u. Threads running on socket %u will use socket %u "
+			"mempool.", socket_id, socket_id, i);
 	  dm->pktmbuf_pools[socket_id] = dm->pktmbuf_pools[i];
 	  return 0;
 	}
@@ -565,6 +686,42 @@ VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
 };
 /* *INDENT-ON* */
 
+#if __x86_64__
+vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx512;
+vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx2;
+vlib_buffer_alloc_from_free_list_cb_t __clib_weak
+  dpdk_buffer_alloc_from_free_list_avx512;
+vlib_buffer_alloc_from_free_list_cb_t __clib_weak
+  dpdk_buffer_alloc_from_free_list_avx2;
+vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_cb_avx512;
+vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_cb_avx2;
+vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_cb_avx512;
+vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_cb_avx2;
+
+static void __clib_constructor
+dpdk_input_multiarch_select (void)
+{
+  vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks;
+  if (dpdk_buffer_alloc_avx512 && clib_cpu_supports_avx512f ())
+    {
+      cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx512;
+      cb->vlib_buffer_alloc_from_free_list_cb =
+	dpdk_buffer_alloc_from_free_list_avx512;
+      cb->vlib_buffer_free_cb = dpdk_buffer_free_cb_avx512;
+      cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_cb_avx512;
+    }
+  else if (dpdk_buffer_alloc_avx2 && clib_cpu_supports_avx2 ())
+    {
+      cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx2;
+      cb->vlib_buffer_alloc_from_free_list_cb =
+	dpdk_buffer_alloc_from_free_list_avx2;
+      cb->vlib_buffer_free_cb = dpdk_buffer_free_cb_avx2;
+      cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_cb_avx2;
+    }
+}
+#endif
+#endif
+
 /** @endcond */
 /*
  * fd.io coding-style-patch-verification: ON