vppinfra: fix issue when copying 16 bytes with clib_memcpy

[vpp.git] / src / vnet / devices / dpdk / buffer.c

diff --git a/src/vnet/devices/dpdk/buffer.c b/src/vnet/devices/dpdk/buffer.c
index 43ceb91..f95d4cb 100644 (file)
--- a/src/vnet/devices/dpdk/buffer.c
+++ b/src/vnet/devices/dpdk/buffer.c
@@ -79,20 +79,46 @@
 STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
               "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
 
+static_always_inline void
+dpdk_rte_pktmbuf_free (vlib_main_t * vm, vlib_buffer_t * b)
+{
+  vlib_buffer_t *hb = b;
+  struct rte_mbuf *mb;
+  u32 next, flags;
+  mb = rte_mbuf_from_vlib_buffer (hb);
+
+next:
+  flags = b->flags;
+  next = b->next_buffer;
+  mb = rte_mbuf_from_vlib_buffer (b);
+
+  if (PREDICT_FALSE (b->n_add_refs))
+    {
+      rte_mbuf_refcnt_update (mb, b->n_add_refs);
+      b->n_add_refs = 0;
+    }
+
+  rte_pktmbuf_free_seg (mb);
+
+  if (flags & VLIB_BUFFER_NEXT_PRESENT)
+    {
+      b = vlib_get_buffer (vm, next);
+      goto next;
+    }
+}
+
 static void
 del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
 {
   u32 i;
-  struct rte_mbuf *mb;
   vlib_buffer_t *b;
 
   for (i = 0; i < vec_len (f->buffers); i++)
     {
       b = vlib_get_buffer (vm, f->buffers[i]);
-      mb = rte_mbuf_from_vlib_buffer (b);
-      ASSERT (rte_mbuf_refcnt_read (mb) == 1);
-      rte_pktmbuf_free (mb);
+      dpdk_rte_pktmbuf_free (vm, b);
     }
+
   vec_free (f->name);
   vec_free (f->buffers);
 }
@@ -140,13 +166,12 @@ fill_free_list (vlib_main_t * vm,
                vlib_buffer_free_list_t * fl, uword min_free_buffers)
 {
   dpdk_main_t *dm = &dpdk_main;
-  vlib_buffer_t *b;
+  vlib_buffer_t *b0, *b1, *b2, *b3;
   int n, i;
-  u32 bi;
-  u32 n_remaining = 0, n_alloc = 0;
+  u32 bi0, bi1, bi2, bi3;
   unsigned socket_id = rte_socket_id ();
   struct rte_mempool *rmp = dm->pktmbuf_pools[socket_id];
-  struct rte_mbuf *mb;
+  struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
 
   /* Too early? */
   if (PREDICT_FALSE (rmp == 0))
@@ -170,24 +195,81 @@ fill_free_list (vlib_main_t * vm,
 
   _vec_len (vm->mbuf_alloc_list) = n;
 
-  for (i = 0; i < n; i++)
+  i = 0;
+
+  while (i < (n - 7))
     {
-      mb = vm->mbuf_alloc_list[i];
+      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
+                                  (vm->mbuf_alloc_list[i + 4]), STORE);
+      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
+                                  (vm->mbuf_alloc_list[i + 5]), STORE);
+      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
+                                  (vm->mbuf_alloc_list[i + 6]), STORE);
+      vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf
+                                  (vm->mbuf_alloc_list[i + 7]), STORE);
+
+      mb0 = vm->mbuf_alloc_list[i];
+      mb1 = vm->mbuf_alloc_list[i + 1];
+      mb2 = vm->mbuf_alloc_list[i + 2];
+      mb3 = vm->mbuf_alloc_list[i + 3];
+
+      ASSERT (rte_mbuf_refcnt_read (mb0) == 0);
+      ASSERT (rte_mbuf_refcnt_read (mb1) == 0);
+      ASSERT (rte_mbuf_refcnt_read (mb2) == 0);
+      ASSERT (rte_mbuf_refcnt_read (mb3) == 0);
+
+      rte_mbuf_refcnt_set (mb0, 1);
+      rte_mbuf_refcnt_set (mb1, 1);
+      rte_mbuf_refcnt_set (mb2, 1);
+      rte_mbuf_refcnt_set (mb3, 1);
+
+      b0 = vlib_buffer_from_rte_mbuf (mb0);
+      b1 = vlib_buffer_from_rte_mbuf (mb1);
+      b2 = vlib_buffer_from_rte_mbuf (mb2);
+      b3 = vlib_buffer_from_rte_mbuf (mb3);
+
+      bi0 = vlib_get_buffer_index (vm, b0);
+      bi1 = vlib_get_buffer_index (vm, b1);
+      bi2 = vlib_get_buffer_index (vm, b2);
+      bi3 = vlib_get_buffer_index (vm, b3);
+
+      vec_add1_aligned (fl->buffers, bi0, CLIB_CACHE_LINE_BYTES);
+      vec_add1_aligned (fl->buffers, bi1, CLIB_CACHE_LINE_BYTES);
+      vec_add1_aligned (fl->buffers, bi2, CLIB_CACHE_LINE_BYTES);
+      vec_add1_aligned (fl->buffers, bi3, CLIB_CACHE_LINE_BYTES);
+
+      vlib_buffer_init_for_free_list (b0, fl);
+      vlib_buffer_init_for_free_list (b1, fl);
+      vlib_buffer_init_for_free_list (b2, fl);
+      vlib_buffer_init_for_free_list (b3, fl);
 
-      ASSERT (rte_mbuf_refcnt_read (mb) == 0);
-      rte_mbuf_refcnt_set (mb, 1);
+      if (fl->buffer_init_function)
+       {
+         fl->buffer_init_function (vm, fl, &bi0, 1);
+         fl->buffer_init_function (vm, fl, &bi1, 1);
+         fl->buffer_init_function (vm, fl, &bi2, 1);
+         fl->buffer_init_function (vm, fl, &bi3, 1);
+       }
+      i += 4;
+    }
+
+  while (i < n)
+    {
+      mb0 = vm->mbuf_alloc_list[i];
 
-      b = vlib_buffer_from_rte_mbuf (mb);
-      bi = vlib_get_buffer_index (vm, b);
+      ASSERT (rte_mbuf_refcnt_read (mb0) == 0);
+      rte_mbuf_refcnt_set (mb0, 1);
 
-      vec_add1_aligned (fl->buffers, bi, CLIB_CACHE_LINE_BYTES);
-      n_alloc++;
-      n_remaining--;
+      b0 = vlib_buffer_from_rte_mbuf (mb0);
+      bi0 = vlib_get_buffer_index (vm, b0);
 
-      vlib_buffer_init_for_free_list (b, fl);
+      vec_add1_aligned (fl->buffers, bi0, CLIB_CACHE_LINE_BYTES);
+
+      vlib_buffer_init_for_free_list (b0, fl);
 
       if (fl->buffer_init_function)
-       fl->buffer_init_function (vm, fl, &bi, 1);
+       fl->buffer_init_function (vm, fl, &bi0, 1);
+      i++;
     }
 
   fl->n_alloc += n;
@@ -269,7 +351,6 @@ vlib_buffer_free_inline (vlib_main_t * vm,
   for (i = 0; i < n_buffers; i++)
     {
       vlib_buffer_t *b;
-      struct rte_mbuf *mb;
 
       b = vlib_get_buffer (vm, buffers[i]);
 
@@ -295,11 +376,7 @@ vlib_buffer_free_inline (vlib_main_t * vm,
       else
        {
          if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
-           {
-             mb = rte_mbuf_from_vlib_buffer (b);
-             ASSERT (rte_mbuf_refcnt_read (mb) == 1);
-             rte_pktmbuf_free (mb);
-           }
+           dpdk_rte_pktmbuf_free (vm, b);
        }
     }
   if (vec_len (bm->announce_list))