X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fbuffer.c;h=cc85235854114f6a3311a7d1756aba8136113a65;hb=b688fb129746c040f5e6f880de592e53aff772d9;hp=6caca72cf03724311f61ad8511814ee472991e6f;hpb=d1274cb207a48f0142a5a6cbcd22d61c22dbc230;p=vpp.git

diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index 6caca72cf03..cc852358541 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -47,10 +47,17 @@
 #include <vlib/unix/unix.h>
 
 vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0;
-static u32 vlib_buffer_physmem_sz = 32 << 20;
+
+/* when running unpriviledged we are limited by RLIMIT_MEMLOCK which is
+   typically set to 16MB so setting default size for buffer memory to 14MB
+   */
+static u32 vlib_buffer_physmem_sz = 14 << 20;
 
 vlib_buffer_main_t buffer_main;
 
+/* logging */
+static vlib_log_class_t buffer_log_default;
+
 uword
 vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
 				       vlib_buffer_t * b_first)
@@ -377,7 +384,7 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm,
 
   pool_get_aligned (vm->buffer_free_list_pool, f, CLIB_CACHE_LINE_BYTES);
 
-  memset (f, 0, sizeof (f[0]));
+  clib_memset (f, 0, sizeof (f[0]));
   f->index = f - vm->buffer_free_list_pool;
   f->n_data_bytes = vlib_buffer_round_size (n_data_bytes);
   f->min_n_buffers_each_alloc = VLIB_FRAME_SIZE;
@@ -439,7 +446,7 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
   vec_free (f->buffers);
 
   /* Poison it. */
-  memset (f, 0xab, sizeof (f[0]));
+  clib_memset (f, 0xab, sizeof (f[0]));
 }
 
 /* Add buffer free list. */
@@ -470,26 +477,10 @@ vlib_buffer_delete_free_list_internal (vlib_main_t * vm,
 }
 
 static_always_inline void *
-vlib_buffer_pool_get_buffer (vlib_buffer_pool_t * bp)
+vlib_buffer_pool_get_buffer (vlib_main_t * vm, vlib_buffer_pool_t * bp)
 {
-  uword slot, page, addr;
-
-  if (PREDICT_FALSE (bp->n_elts == bp->n_used))
-    {
-      clib_spinlock_unlock (&bp->lock);
-      return 0;
-    }
-  slot = bp->next_clear;
-  bp->bitmap = clib_bitmap_set (bp->bitmap, slot, 1);
-  bp->next_clear = clib_bitmap_next_clear (bp->bitmap, slot + 1);
-  bp->n_used++;
-
-  page = slot / bp->buffers_per_page;
-  slot -= page * bp->buffers_per_page;
-
-  addr = bp->start + (page << bp->log2_page_size) + slot * bp->buffer_size;
-
-  return uword_to_pointer (addr, void *);
+  return vlib_physmem_alloc_from_map (vm, bp->physmem_map_index,
+				      bp->buffer_size, CLIB_CACHE_LINE_BYTES);
 }
 
 /* Make sure free list has at least given number of free buffers. */
@@ -533,7 +524,7 @@ vlib_buffer_fill_free_list_internal (vlib_main_t * vm,
   clib_spinlock_lock (&bp->lock);
   while (n_alloc < n)
     {
-      if ((b = vlib_buffer_pool_get_buffer (bp)) == 0)
+      if ((b = vlib_buffer_pool_get_buffer (vm, bp)) == 0)
 	goto done;
 
       n_alloc += 1;
@@ -544,7 +535,7 @@ vlib_buffer_fill_free_list_internal (vlib_main_t * vm,
       if (CLIB_DEBUG > 0)
 	vlib_buffer_set_known_state (bi[0], VLIB_BUFFER_KNOWN_FREE);
 
-      memset (b, 0, sizeof (vlib_buffer_t));
+      clib_memset (b, 0, sizeof (vlib_buffer_t));
       vlib_buffer_init_for_free_list (b, fl);
 
       if (fl->buffer_init_function)
@@ -573,50 +564,26 @@ recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi,
 {
   vlib_buffer_free_list_t *fl;
   vlib_buffer_free_list_index_t fi;
-  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
+  u32 flags, next;
 
-  /* The only current use of this callback:
-   * multicast recycle */
-  if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
-    {
-      int j;
+  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
 
-      vlib_buffer_add_to_free_list (vm, fl, bi,
-				    (b->flags & VLIB_BUFFER_RECYCLE) == 0);
-      for (j = 0; j < vec_len (vm->buffer_announce_list); j++)
-	{
-	  if (fl == vm->buffer_announce_list[j])
-	    goto already_announced;
-	}
-      vec_add1 (vm->buffer_announce_list, fl);
-    already_announced:
-      ;
-    }
-  else
+  do
     {
-      if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
+      vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
+      flags = nb->flags;
+      next = nb->next_buffer;
+      if (nb->n_add_refs)
+	nb->n_add_refs--;
+      else
 	{
-	  u32 flags, next;
-
-	  do
-	    {
-	      vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
-	      flags = nb->flags;
-	      next = nb->next_buffer;
-	      if (nb->n_add_refs)
-		nb->n_add_refs--;
-	      else
-		{
-		  vlib_buffer_validate_alloc_free (vm, &bi, 1,
-						   VLIB_BUFFER_KNOWN_ALLOCATED);
-		  vlib_buffer_add_to_free_list (vm, fl, bi, 1);
-		}
-	      bi = next;
-	    }
-	  while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
-
+	  vlib_buffer_validate_alloc_free (vm, &bi, 1,
+					   VLIB_BUFFER_KNOWN_ALLOCATED);
+	  vlib_buffer_add_to_free_list (vm, fl, bi, 1);
 	}
+      bi = next;
     }
+  while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
 }
 
 static_always_inline void
@@ -673,17 +640,6 @@ vlib_buffer_free_inline (vlib_main_t * vm,
       recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
       i++;
     }
-
-  if (vec_len (vm->buffer_announce_list))
-    {
-      vlib_buffer_free_list_t *fl;
-      for (i = 0; i < vec_len (vm->buffer_announce_list); i++)
-	{
-	  fl = vm->buffer_announce_list[i];
-	  fl->buffers_added_to_freelist_function (vm, fl);
-	}
-      _vec_len (vm->buffer_announce_list) = 0;
-    }
 }
 
 static void
@@ -744,7 +700,7 @@ vlib_packet_template_init (vlib_main_t * vm,
 
   vlib_worker_thread_barrier_sync (vm);
 
-  memset (t, 0, sizeof (t[0]));
+  clib_memset (t, 0, sizeof (t[0]));
 
   vec_add (t->packet_data, packet_data, n_packet_data_bytes);
   t->min_n_buffers_each_alloc = min_n_buffers_each_alloc;
@@ -817,7 +773,7 @@ vlib_buffer_add_data (vlib_main_t * vm,
   void *d;
 
   bi = buffer_index;
-  if (bi == 0
+  if (bi == ~0
       && 1 != vlib_buffer_alloc_from_free_list (vm, &bi, 1, free_list_index))
     goto out_of_buffers;
 
@@ -886,7 +842,7 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
 	      vlib_buffer_alloc_from_free_list (vm, &l->next_buffer, 1,
 						free_list_index))
 	    return copied;
-	  *last = l = vlib_buffer_chain_buffer (vm, first, l, l->next_buffer);
+	  *last = l = vlib_buffer_chain_buffer (vm, l, l->next_buffer);
 	  max = n_buffer_bytes - l->current_length - l->current_data;
 	}
 
@@ -901,14 +857,13 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
 }
 
 u8
-vlib_buffer_pool_create (vlib_main_t * vm, vlib_physmem_region_index_t pri,
-			 u16 buffer_size)
+vlib_buffer_register_physmem_map (vlib_main_t * vm, u32 physmem_map_index)
 {
   vlib_buffer_main_t *bm = &buffer_main;
-  vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, pri);
   vlib_buffer_pool_t *p;
-  uword start = pointer_to_uword (pr->mem);
-  uword size = pr->size;
+  vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index);
+  uword start = pointer_to_uword (m->base);
+  uword size = (uword) m->n_pages << m->log2_page_size;
 
   if (bm->buffer_mem_size == 0)
     {
@@ -938,18 +893,8 @@ vlib_buffer_pool_create (vlib_main_t * vm, vlib_physmem_region_index_t pri,
   vec_add2 (bm->buffer_pools, p, 1);
   p->start = start;
   p->size = size;
-  p->physmem_region = pri;
+  p->physmem_map_index = physmem_map_index;
 
-  if (buffer_size == 0)
-    goto done;
-
-  p->log2_page_size = pr->log2_page_size;
-  p->buffer_size = buffer_size;
-  p->buffers_per_page = (1 << pr->log2_page_size) / p->buffer_size;
-  p->n_elts = p->buffers_per_page * pr->n_pages;
-  p->n_used = 0;
-  clib_spinlock_init (&p->lock);
-done:
   ASSERT (p - bm->buffer_pools < 256);
   return p - bm->buffer_pools;
 }
@@ -1018,8 +963,12 @@ clib_error_t *
 vlib_buffer_main_init (struct vlib_main_t * vm)
 {
   vlib_buffer_main_t *bm = &buffer_main;
-  vlib_physmem_region_index_t pri;
   clib_error_t *error;
+  u32 physmem_map_index;
+  u8 pool_index;
+  int log2_page_size = 0;
+
+  buffer_log_default = vlib_log_register_class ("buffer", 0);
 
   if (vlib_buffer_callbacks)
     {
@@ -1038,25 +987,33 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
     &vlib_buffer_delete_free_list_internal;
   clib_spinlock_init (&bm->buffer_known_hash_lockp);
 
-  /* allocate default region */
-  error = vlib_physmem_region_alloc (vm, "buffers",
-				     vlib_buffer_physmem_sz, 0,
-				     VLIB_PHYSMEM_F_SHARED |
-				     VLIB_PHYSMEM_F_HUGETLB, &pri);
+retry:
+  error = vlib_physmem_shared_map_create (vm, "buffers",
+					  vlib_buffer_physmem_sz,
+					  log2_page_size,
+					  CLIB_PMALLOC_NUMA_LOCAL,
+					  &physmem_map_index);
+
+  if (error && log2_page_size == 0)
+    {
+      vlib_log_warn (buffer_log_default, "%U", format_clib_error, error);
+      clib_error_free (error);
+      vlib_log_warn (buffer_log_default, "falling back to non-hugepage "
+		     "backed buffer pool");
+      log2_page_size = min_log2 (clib_mem_get_page_size ());
+      goto retry;
+    }
 
-  if (error == 0)
-    goto done;
+  if (error)
+    return error;
 
-  clib_error_free (error);
+  pool_index = vlib_buffer_register_physmem_map (vm, physmem_map_index);
+  vlib_buffer_pool_t *bp = vlib_buffer_pool_get (pool_index);
+  clib_spinlock_init (&bp->lock);
+  bp->buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES +
+    sizeof (vlib_buffer_t);
 
-  error = vlib_physmem_region_alloc (vm, "buffers",
-				     vlib_buffer_physmem_sz, 0,
-				     VLIB_PHYSMEM_F_SHARED, &pri);
-done:
-  if (error == 0)
-    vlib_buffer_pool_create (vm, pri, sizeof (vlib_buffer_t) +
-			     VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES);
-  return error;
+  return 0;
 }
 
 static clib_error_t *