X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fbuffer.c;h=f5b813793a31697d02a726a7d6233682841edf92;hb=c74b43c80789f5e437dfe4cc491157b45a7f222e;hp=77602a61ac19bff75ae0553af53f97ff14a363a0;hpb=e82c73a96ea68268804b3f8b3db8cca738cbfe64;p=vpp.git

diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index 77602a61ac1..f5b813793a3 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -46,6 +46,7 @@
 #include <vppinfra/linux/sysfs.h>
 #include <vlib/vlib.h>
 #include <vlib/unix/unix.h>
+#include <vpp/stats/stat_segment.h>
 
 #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA 16384
 #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV 8192
@@ -62,6 +63,17 @@ STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64);
 
 u16 __vlib_buffer_external_hdr_size = 0;
 
+static void
+buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e,
+				u32 index);
+
+static void
+buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e,
+				   u32 index);
+
+static void
+buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index);
+
 uword
 vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
 				       vlib_buffer_t * b_first)
@@ -80,7 +92,7 @@ vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
 }
 
 u8 *
-format_vlib_buffer (u8 * s, va_list * args)
+format_vlib_buffer_no_chain (u8 * s, va_list * args)
 {
   vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
   u32 indent = format_get_indent (s);
@@ -100,15 +112,26 @@ format_vlib_buffer (u8 * s, va_list * args)
 		b->total_length_not_including_first_buffer);
 
   if (b->flags & VLIB_BUFFER_IS_TRACED)
-    s = format (s, ", trace 0x%x", b->trace_index);
+    s = format (s, ", trace handle 0x%x", b->trace_handle);
 
   if (a)
     s = format (s, "\n%U%v", format_white_space, indent, a);
   vec_free (a);
 
+  return s;
+}
+
+u8 *
+format_vlib_buffer (u8 * s, va_list * args)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
+  u32 indent = format_get_indent (s);
+
+  s = format (s, "%U", format_vlib_buffer_no_chain, b);
+
   while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
     {
-      vlib_main_t *vm = vlib_get_main ();
       u32 next_buffer = b->next_buffer;
       b = vlib_get_buffer (vm, next_buffer);
 
@@ -192,9 +215,10 @@ vlib_validate_buffer_helper (vlib_main_t * vm,
   if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE)
     return format (0, "current data %d before pre-data", b->current_data);
 
-  if (b->current_data + b->current_length > VLIB_BUFFER_DATA_SIZE)
+  if (b->current_data + b->current_length >
+      vlib_buffer_get_default_data_size (vm))
     return format (0, "%d-%d beyond end of buffer %d", b->current_data,
-		   b->current_length, VLIB_BUFFER_DATA_SIZE);
+		   b->current_length, vlib_buffer_get_default_data_size (vm));
 
   if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
     {
@@ -308,7 +332,7 @@ static struct
 vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
 
 
-/* When dubugging validate that given buffers are either known allocated
+/* When debugging validate that given buffers are either known allocated
    or known free. */
 void
 vlib_buffer_validate_alloc_free (vlib_main_t * vm,
@@ -407,7 +431,7 @@ vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
 
   d = data;
   n_left = n_data_bytes;
-  n_buffer_bytes = VLIB_BUFFER_DATA_SIZE;
+  n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
 
   b = vlib_get_buffer (vm, bi);
   b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
@@ -455,7 +479,7 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
 					  u16 data_len)
 {
   vlib_buffer_t *l = *last;
-  u32 n_buffer_bytes = VLIB_BUFFER_DATA_SIZE;
+  u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
   u16 copied = 0;
   ASSERT (n_buffer_bytes >= l->current_length + l->current_data);
   while (data_len)
@@ -480,26 +504,22 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
   return copied;
 }
 
-clib_error_t *
-vlib_buffer_pool_create (vlib_main_t * vm, u8 index, char *name,
-			 u32 data_size, u32 physmem_map_index)
+u8
+vlib_buffer_pool_create (vlib_main_t * vm, char *name, u32 data_size,
+			 u32 physmem_map_index)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
   vlib_buffer_pool_t *bp;
   vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index);
   uword start = pointer_to_uword (m->base);
   uword size = (uword) m->n_pages << m->log2_page_size;
-  u32 alloc_size, n_alloc_per_page;;
-
-  vec_validate_aligned (bm->buffer_pools, index, CLIB_CACHE_LINE_BYTES);
-  bp = vec_elt_at_index (bm->buffer_pools, index);
+  uword i, j;
+  u32 alloc_size, n_alloc_per_page;
 
-  if (bp->start)
-    return clib_error_return (0, "buffer with index %u already exists",
-			      index);
+  if (vec_len (bm->buffer_pools) >= 255)
+    return ~0;
 
-  if (index >= 255)
-    return clib_error_return (0, "buffer index must be < 255", index);
+  vec_add2_aligned (bm->buffer_pools, bp, 1, CLIB_LOG2_CACHE_LINE_BYTES);
 
   if (bm->buffer_mem_size == 0)
     {
@@ -543,36 +563,39 @@ vlib_buffer_pool_create (vlib_main_t * vm, u8 index, char *name,
   n_alloc_per_page = (1ULL << m->log2_page_size) / alloc_size;
 
   /* preallocate buffer indices memory */
-  vec_validate_aligned (bp->buffers, m->n_pages * n_alloc_per_page,
-			CLIB_CACHE_LINE_BYTES);
-  vec_reset_length (bp->buffers);
+  bp->n_buffers = m->n_pages * n_alloc_per_page;
+  bp->buffers = clib_mem_alloc_aligned (bp->n_buffers * sizeof (u32),
+					CLIB_CACHE_LINE_BYTES);
 
   clib_spinlock_init (&bp->lock);
 
-  while (1)
-    {
-      u8 *p;
-      u32 bi;
+  for (j = 0; j < m->n_pages; j++)
+    for (i = 0; i < n_alloc_per_page; i++)
+      {
+	u8 *p;
+	u32 bi;
 
-      p = vlib_physmem_alloc_from_map (vm, bp->physmem_map_index, alloc_size,
-				       CLIB_CACHE_LINE_BYTES);
+	p = m->base + (j << m->log2_page_size) + i * alloc_size;
+	p += bm->ext_hdr_size;
 
-      if (p == 0)
-	break;
-
-      p += bm->ext_hdr_size;
+	/*
+	 * Waste 1 buffer (maximum) so that 0 is never a valid buffer index.
+	 * Allows various places to ASSERT (bi != 0). Much easier
+	 * than debugging downstream crashes in successor nodes.
+	 */
+	if (p == m->base)
+	  continue;
 
-      vlib_buffer_copy_template ((vlib_buffer_t *) p, &bp->buffer_template);
+	vlib_buffer_copy_template ((vlib_buffer_t *) p, &bp->buffer_template);
 
-      bi = vlib_get_buffer_index (vm, (vlib_buffer_t *) p);
+	bi = vlib_get_buffer_index (vm, (vlib_buffer_t *) p);
 
-      vec_add1_aligned (bp->buffers, bi, CLIB_CACHE_LINE_BYTES);
-      vlib_get_buffer (vm, bi);
+	bp->buffers[bp->n_avail++] = bi;
 
-      bp->n_buffers += 1;
-    }
+	vlib_get_buffer (vm, bi);
+      }
 
-  return 0;
+  return bp->index;
 }
 
 static u8 *
@@ -590,14 +613,14 @@ format_vlib_buffer_pool (u8 * s, va_list * va)
 
   /* *INDENT-OFF* */
   vec_foreach (bpt, bp->threads)
-    cached += vec_len (bpt->cached_buffers);
+    cached += bpt->n_cached;
   /* *INDENT-ON* */
 
   s = format (s, "%-20s%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u",
 	      bp->name, bp->index, bp->numa_node, bp->data_size +
 	      sizeof (vlib_buffer_t) + vm->buffer_main->ext_hdr_size,
-	      bp->data_size, bp->n_buffers, vec_len (bp->buffers), cached,
-	      bp->n_buffers - vec_len (bp->buffers) - cached);
+	      bp->data_size, bp->n_buffers, bp->n_avail, cached,
+	      bp->n_buffers - bp->n_avail - cached);
 
   return s;
 }
@@ -649,7 +672,8 @@ vlib_buffer_worker_init (vlib_main_t * vm)
 VLIB_WORKER_INIT_FUNCTION (vlib_buffer_worker_init);
 
 static clib_error_t *
-vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node)
+vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node,
+				 u8 * index)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
   clib_error_t *error;
@@ -658,7 +682,8 @@ vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node)
   u32 buffers_per_numa;
   u32 buffer_size = CLIB_CACHE_LINE_ROUND (bm->ext_hdr_size +
 					   sizeof (vlib_buffer_t) +
-					   VLIB_BUFFER_DATA_SIZE);
+					   vlib_buffer_get_default_data_size
+					   (vm));
   u8 *name;
 
   pagesize = clib_mem_get_default_hugepage_size ();
@@ -687,28 +712,113 @@ retry:
     }
 
   if (error)
-    return error;
+    goto done;
 
   vec_reset_length (name);
   name = format (name, "default-numa-%d%c", numa_node, 0);
 
-  return vlib_buffer_pool_create (vm, numa_node, (char *) name,
-				  VLIB_BUFFER_DATA_SIZE, physmem_map_index);
+  *index = vlib_buffer_pool_create (vm, (char *) name,
+				    vlib_buffer_get_default_data_size (vm),
+				    physmem_map_index);
+
+  if (*index == (u8) ~ 0)
+    {
+      error = clib_error_return (0, "maximum number of buffer pools reached");
+      goto done;
+    }
+
+done:
+  vec_free (name);
+  return error;
+}
+
+void
+vlib_buffer_main_alloc (vlib_main_t * vm)
+{
+  vlib_buffer_main_t *bm;
+
+  if (vm->buffer_main)
+    return;
+
+  vm->buffer_main = bm = clib_mem_alloc (sizeof (bm[0]));
+  clib_memset (vm->buffer_main, 0, sizeof (bm[0]));
+  bm->default_data_size = VLIB_BUFFER_DEFAULT_DATA_SIZE;
+}
+
+static u32
+buffer_get_cached (vlib_buffer_pool_t * bp)
+{
+  u32 cached = 0;
+  vlib_buffer_pool_thread_t *bpt;
+
+  clib_spinlock_lock (&bp->lock);
+
+  /* *INDENT-OFF* */
+  vec_foreach (bpt, bp->threads)
+    cached += bpt->n_cached;
+  /* *INDENT-ON* */
+
+  clib_spinlock_unlock (&bp->lock);
+
+  return cached;
+}
+
+static vlib_buffer_pool_t *
+buffer_get_by_index (vlib_buffer_main_t * bm, u32 index)
+{
+  vlib_buffer_pool_t *bp;
+  if (!bm->buffer_pools || vec_len (bm->buffer_pools) < index)
+    return 0;
+  bp = vec_elt_at_index (bm->buffer_pools, index);
+
+  return bp;
+}
+
+static void
+buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index);
+  if (!bp)
+    return;
+
+  e->value = bp->n_buffers - bp->n_avail - buffer_get_cached (bp);
+}
+
+static void
+buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e,
+				   u32 index)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index);
+  if (!bp)
+    return;
+
+  e->value = bp->n_avail;
+}
+
+static void
+buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e, u32 index)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index);
+  if (!bp)
+    return;
+
+  e->value = buffer_get_cached (bp);
 }
 
 clib_error_t *
-vlib_buffer_main_init (struct vlib_main_t *vm)
+vlib_buffer_main_init (struct vlib_main_t * vm)
 {
   vlib_buffer_main_t *bm;
   clib_error_t *err;
-  clib_bitmap_t *bmp = 0;
+  clib_bitmap_t *bmp = 0, *bmp_has_memory = 0;
   u32 numa_node;
+  vlib_buffer_pool_t *bp;
+  u8 *name = 0, first_valid_buffer_pool_index = ~0;
 
-  if (vm->buffer_main == 0)
-    {
-      vm->buffer_main = clib_mem_alloc (sizeof (bm[0]));
-      clib_memset (vm->buffer_main, 0, sizeof (bm[0]));
-    }
+  vlib_buffer_main_alloc (vm);
 
   bm = vm->buffer_main;
   bm->log_default = vlib_log_register_class ("buffer", 0);
@@ -716,26 +826,82 @@ vlib_buffer_main_init (struct vlib_main_t *vm)
 
   clib_spinlock_init (&bm->buffer_known_hash_lockp);
 
-  err = clib_sysfs_read ("/sys/devices/system/node/possible", "%U",
-			 unformat_bitmap_list, &bmp);
-  if (err)
+  if ((err = clib_sysfs_read ("/sys/devices/system/node/online", "%U",
+			      unformat_bitmap_list, &bmp)))
+    clib_error_free (err);
+
+  if ((err = clib_sysfs_read ("/sys/devices/system/node/has_memory", "%U",
+			      unformat_bitmap_list, &bmp_has_memory)))
+    clib_error_free (err);
+
+  if (bmp && bmp_has_memory)
+    bmp = clib_bitmap_and (bmp, bmp_has_memory);
+
+  /* no info from sysfs, assuming that only numa 0 exists */
+  if (bmp == 0)
+    bmp = clib_bitmap_set (bmp, 0, 1);
+
+  if (clib_bitmap_last_set (bmp) >= VLIB_BUFFER_MAX_NUMA_NODES)
+    clib_panic ("system have more than %u NUMA nodes",
+		VLIB_BUFFER_MAX_NUMA_NODES);
+
+  /* *INDENT-OFF* */
+  clib_bitmap_foreach (numa_node, bmp,
+    {
+      u8 *index = bm->default_buffer_pool_index_for_numa + numa_node;
+      index[0] = ~0;
+      if ((err = vlib_buffer_main_init_numa_node (vm, numa_node, index)))
+        {
+	  clib_error_report (err);
+	  clib_error_free (err);
+	  continue;
+	}
+
+      if (first_valid_buffer_pool_index == 0xff)
+        first_valid_buffer_pool_index = index[0];
+    });
+  /* *INDENT-ON* */
+
+  if (first_valid_buffer_pool_index == (u8) ~ 0)
     {
-      /* no info from sysfs, assuming that only numa 0 exists */
-      clib_error_free (err);
-      bmp = clib_bitmap_set (bmp, 0, 1);
+      err = clib_error_return (0, "failed to allocate buffer pool(s)");
+      goto done;
     }
 
   /* *INDENT-OFF* */
-  clib_bitmap_foreach (numa_node, bmp, {
-      if ((err = vlib_buffer_main_init_numa_node(vm, numa_node)))
-	  goto done;
+  clib_bitmap_foreach (numa_node, bmp,
+    {
+      if (bm->default_buffer_pool_index_for_numa[numa_node]  == (u8) ~0)
+	bm->default_buffer_pool_index_for_numa[numa_node] =
+	  first_valid_buffer_pool_index;
     });
   /* *INDENT-ON* */
 
-  bm->n_numa_nodes = clib_bitmap_last_set (bmp) + 1;
+  vec_foreach (bp, bm->buffer_pools)
+  {
+    if (bp->n_buffers == 0)
+      continue;
+
+    vec_reset_length (name);
+    name = format (name, "/buffer-pools/%s/cached%c", bp->name, 0);
+    stat_segment_register_gauge (name, buffer_gauges_update_cached_fn,
+				 bp - bm->buffer_pools);
+
+    vec_reset_length (name);
+    name = format (name, "/buffer-pools/%s/used%c", bp->name, 0);
+    stat_segment_register_gauge (name, buffer_gauges_update_used_fn,
+				 bp - bm->buffer_pools);
+
+    vec_reset_length (name);
+    name = format (name, "/buffer-pools/%s/available%c", bp->name, 0);
+    stat_segment_register_gauge (name, buffer_gauges_update_available_fn,
+				 bp - bm->buffer_pools);
+  }
 
 done:
   vec_free (bmp);
+  vec_free (bmp_has_memory);
+  vec_free (name);
   return err;
 }
 
@@ -744,11 +910,7 @@ vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input)
 {
   vlib_buffer_main_t *bm;
 
-  if (vm->buffer_main == 0)
-    {
-      vm->buffer_main = clib_mem_alloc (sizeof (bm[0]));
-      clib_memset (vm->buffer_main, 0, sizeof (bm[0]));
-    }
+  vlib_buffer_main_alloc (vm);
 
   bm = vm->buffer_main;
 
@@ -756,6 +918,9 @@ vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input)
     {
       if (unformat (input, "buffers-per-numa %u", &bm->buffers_per_numa))
 	;
+      else if (unformat (input, "default data-size %u",
+			 &bm->default_data_size))
+	;
       else
 	return unformat_parse_error (input);
     }
@@ -766,6 +931,24 @@ vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input)
 
 VLIB_EARLY_CONFIG_FUNCTION (vlib_buffers_configure, "buffers");
 
+#if VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0
+u32
+vlib_buffer_alloc_may_fail (vlib_main_t * vm, u32 n_buffers)
+{
+  f64 r;
+
+  r = random_f64 (&vm->buffer_alloc_success_seed);
+
+  /* Fail this request? */
+  if (r > vm->buffer_alloc_success_rate)
+    n_buffers--;
+  /* 5% chance of returning nothing at all */
+  if (r > vm->buffer_alloc_success_rate && r > 0.95)
+    n_buffers = 0;
+
+  return n_buffers;
+}
+#endif
 
 /** @endcond */
 /*