X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fbuffer.c;h=f8cfb0357faa83ddfd4fa01fc94282ccd2cb097f;hb=6ffb7c61899eb782416cbd2f117042a583f189f7;hp=0e972caadc9258a9fc0e0185cecd58d3afb71b1d;hpb=b592d1b64140e248f91dc76ce17da65ea90a4798;p=vpp.git

diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index 0e972caadc9..f8cfb0357fa 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -92,7 +92,7 @@ vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
 }
 
 u8 *
-format_vlib_buffer (u8 * s, va_list * args)
+format_vlib_buffer_no_chain (u8 * s, va_list * args)
 {
   vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
   u32 indent = format_get_indent (s);
@@ -112,15 +112,26 @@ format_vlib_buffer (u8 * s, va_list * args)
 		b->total_length_not_including_first_buffer);
 
   if (b->flags & VLIB_BUFFER_IS_TRACED)
-    s = format (s, ", trace 0x%x", b->trace_index);
+    s = format (s, ", trace handle 0x%x", b->trace_handle);
 
   if (a)
     s = format (s, "\n%U%v", format_white_space, indent, a);
   vec_free (a);
 
+  return s;
+}
+
+u8 *
+format_vlib_buffer (u8 * s, va_list * args)
+{
+  vlib_main_t *vm = vlib_get_main ();
+  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
+  u32 indent = format_get_indent (s);
+
+  s = format (s, "%U", format_vlib_buffer_no_chain, b);
+
   while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
     {
-      vlib_main_t *vm = vlib_get_main ();
       u32 next_buffer = b->next_buffer;
       b = vlib_get_buffer (vm, next_buffer);
 
@@ -321,7 +332,7 @@ static struct
 vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
 
 
-/* When dubugging validate that given buffers are either known allocated
+/* When debugging validate that given buffers are either known allocated
    or known free. */
 void
 vlib_buffer_validate_alloc_free (vlib_main_t * vm,
@@ -493,6 +504,19 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
   return copied;
 }
 
+static uword
+vlib_buffer_alloc_size (uword ext_hdr_size, uword data_size)
+{
+  uword alloc_size = ext_hdr_size + sizeof (vlib_buffer_t) + data_size;
+  alloc_size = CLIB_CACHE_LINE_ROUND (alloc_size);
+
+  /* in case when we have even number of cachelines, we add one more for
+   * better cache occupancy */
+  alloc_size |= CLIB_CACHE_LINE_BYTES;
+
+  return alloc_size;
+}
+
 u8
 vlib_buffer_pool_create (vlib_main_t * vm, char *name, u32 data_size,
 			 u32 physmem_map_index)
@@ -545,16 +569,16 @@ vlib_buffer_pool_create (vlib_main_t * vm, char *name, u32 data_size,
   bp->data_size = data_size;
   bp->numa_node = m->numa_node;
 
-  vec_validate_aligned (bp->threads, vec_len (vlib_mains) - 1,
+  vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
 			CLIB_CACHE_LINE_BYTES);
 
-  alloc_size = data_size + sizeof (vlib_buffer_t) + bm->ext_hdr_size;
+  alloc_size = vlib_buffer_alloc_size (bm->ext_hdr_size, data_size);
   n_alloc_per_page = (1ULL << m->log2_page_size) / alloc_size;
 
   /* preallocate buffer indices memory */
-  vec_validate_aligned (bp->buffers, m->n_pages * n_alloc_per_page,
-			CLIB_CACHE_LINE_BYTES);
-  vec_reset_length (bp->buffers);
+  bp->n_buffers = m->n_pages * n_alloc_per_page;
+  bp->buffers = clib_mem_alloc_aligned (bp->n_buffers * sizeof (u32),
+					CLIB_CACHE_LINE_BYTES);
 
   clib_spinlock_init (&bp->lock);
 
@@ -567,15 +591,23 @@ vlib_buffer_pool_create (vlib_main_t * vm, char *name, u32 data_size,
 	p = m->base + (j << m->log2_page_size) + i * alloc_size;
 	p += bm->ext_hdr_size;
 
+	/*
+	 * Waste 1 buffer (maximum) so that 0 is never a valid buffer index.
+	 * Allows various places to ASSERT (bi != 0). Much easier
+	 * than debugging downstream crashes in successor nodes.
+	 */
+	if (p == m->base)
+	  continue;
+
 	vlib_buffer_copy_template ((vlib_buffer_t *) p, &bp->buffer_template);
 
 	bi = vlib_get_buffer_index (vm, (vlib_buffer_t *) p);
 
-	vec_add1_aligned (bp->buffers, bi, CLIB_CACHE_LINE_BYTES);
+	bp->buffers[bp->n_avail++] = bi;
+
 	vlib_get_buffer (vm, bi);
       }
 
-  bp->n_buffers = vec_len (bp->buffers);
   return bp->index;
 }
 
@@ -594,14 +626,14 @@ format_vlib_buffer_pool (u8 * s, va_list * va)
 
   /* *INDENT-OFF* */
   vec_foreach (bpt, bp->threads)
-    cached += vec_len (bpt->cached_buffers);
+    cached += bpt->n_cached;
   /* *INDENT-ON* */
 
   s = format (s, "%-20s%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u",
 	      bp->name, bp->index, bp->numa_node, bp->data_size +
 	      sizeof (vlib_buffer_t) + vm->buffer_main->ext_hdr_size,
-	      bp->data_size, bp->n_buffers, vec_len (bp->buffers), cached,
-	      bp->n_buffers - vec_len (bp->buffers) - cached);
+	      bp->data_size, bp->n_buffers, bp->n_avail, cached,
+	      bp->n_buffers - bp->n_avail - cached);
 
   return s;
 }
@@ -641,7 +673,7 @@ vlib_buffer_worker_init (vlib_main_t * vm)
   vec_foreach (bp, bm->buffer_pools)
     {
       clib_spinlock_lock (&bp->lock);
-      vec_validate_aligned (bp->threads, vec_len (vlib_mains) - 1,
+      vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
 			    CLIB_CACHE_LINE_BYTES);
       clib_spinlock_unlock (&bp->lock);
     }
@@ -653,59 +685,91 @@ vlib_buffer_worker_init (vlib_main_t * vm)
 VLIB_WORKER_INIT_FUNCTION (vlib_buffer_worker_init);
 
 static clib_error_t *
-vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node,
-				 u8 * index)
+vlib_buffer_main_init_numa_alloc (struct vlib_main_t *vm, u32 numa_node,
+				  u32 * physmem_map_index,
+				  clib_mem_page_sz_t log2_page_size,
+				  u8 unpriv)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
+  u32 buffers_per_numa = bm->buffers_per_numa;
   clib_error_t *error;
-  u32 physmem_map_index;
+  u32 buffer_size;
   uword n_pages, pagesize;
-  u32 buffers_per_numa;
-  u32 buffer_size = CLIB_CACHE_LINE_ROUND (bm->ext_hdr_size +
-					   sizeof (vlib_buffer_t) +
-					   vlib_buffer_get_default_data_size
-					   (vm));
-  u8 *name;
-
-  pagesize = clib_mem_get_default_hugepage_size ();
-  name = format (0, "buffers-numa-%d%c", numa_node, 0);
+  u8 *name = 0;
 
-  buffers_per_numa = bm->buffers_per_numa ? bm->buffers_per_numa :
-    VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA;
+  ASSERT (log2_page_size != CLIB_MEM_PAGE_SZ_UNKNOWN);
 
-retry:
+  pagesize = clib_mem_page_bytes (log2_page_size);
+  buffer_size = vlib_buffer_alloc_size (bm->ext_hdr_size,
+					vlib_buffer_get_default_data_size
+					(vm));
+  if (buffer_size > pagesize)
+    return clib_error_return (0, "buffer size (%llu) is greater than page "
+			      "size (%llu)", buffer_size, pagesize);
+
+  if (buffers_per_numa == 0)
+    buffers_per_numa = unpriv ? VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV :
+      VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA;
+
+  name = format (0, "buffers-numa-%d%c", numa_node, 0);
   n_pages = (buffers_per_numa - 1) / (pagesize / buffer_size) + 1;
   error = vlib_physmem_shared_map_create (vm, (char *) name,
 					  n_pages * pagesize,
 					  min_log2 (pagesize), numa_node,
-					  &physmem_map_index);
+					  physmem_map_index);
+  vec_free (name);
+  return error;
+}
+
+static clib_error_t *
+vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node,
+				 u8 * index)
+{
+  vlib_buffer_main_t *bm = vm->buffer_main;
+  u32 physmem_map_index;
+  clib_error_t *error;
+  u8 *name = 0;
 
-  if (error && pagesize != clib_mem_get_page_size ())
+  if (bm->log2_page_size == CLIB_MEM_PAGE_SZ_UNKNOWN)
     {
-      vlib_log_warn (bm->log_default, "%U", format_clib_error, error);
+      error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
+						&physmem_map_index,
+						CLIB_MEM_PAGE_SZ_DEFAULT_HUGE,
+						0 /* unpriv */ );
+      if (!error)
+	goto buffer_pool_create;
+
+      /* If alloc failed, retry without hugepages */
+      vlib_log_warn (bm->log_default,
+		     "numa[%u] falling back to non-hugepage backed "
+		     "buffer pool (%U)", numa_node, format_clib_error, error);
       clib_error_free (error);
-      vlib_log_warn (bm->log_default, "falling back to non-hugepage "
-		     "backed buffer pool");
-      pagesize = clib_mem_get_page_size ();
-      buffers_per_numa = bm->buffers_per_numa ? bm->buffers_per_numa :
-	VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV;
-      goto retry;
-    }
 
+      error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
+						&physmem_map_index,
+						CLIB_MEM_PAGE_SZ_DEFAULT,
+						1 /* unpriv */ );
+    }
+  else
+    error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
+					      &physmem_map_index,
+					      bm->log2_page_size,
+					      0 /* unpriv */ );
   if (error)
     return error;
 
-  vec_reset_length (name);
+buffer_pool_create:
   name = format (name, "default-numa-%d%c", numa_node, 0);
-
   *index = vlib_buffer_pool_create (vm, (char *) name,
 				    vlib_buffer_get_default_data_size (vm),
 				    physmem_map_index);
 
   if (*index == (u8) ~ 0)
-    return clib_error_return (0, "maximum number of buffer pools reached");
+    error = clib_error_return (0, "maximum number of buffer pools reached");
+  vec_free (name);
 
-  return 0;
+
+  return error;
 }
 
 void
@@ -731,7 +795,7 @@ buffer_get_cached (vlib_buffer_pool_t * bp)
 
   /* *INDENT-OFF* */
   vec_foreach (bpt, bp->threads)
-    cached += vec_len (bpt->cached_buffers);
+    cached += bpt->n_cached;
   /* *INDENT-ON* */
 
   clib_spinlock_unlock (&bp->lock);
@@ -746,8 +810,7 @@ buffer_get_by_index (vlib_buffer_main_t * bm, u32 index)
   if (!bm->buffer_pools || vec_len (bm->buffer_pools) < index)
     return 0;
   bp = vec_elt_at_index (bm->buffer_pools, index);
-  if (!bp)
-    return 0;
+
   return bp;
 }
 
@@ -759,7 +822,7 @@ buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index)
   if (!bp)
     return;
 
-  e->value = bp->n_buffers - vec_len (bp->buffers) - buffer_get_cached (bp);
+  e->value = bp->n_buffers - bp->n_avail - buffer_get_cached (bp);
 }
 
 static void
@@ -771,7 +834,7 @@ buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e,
   if (!bp)
     return;
 
-  e->value = vec_len (bp->buffers);
+  e->value = bp->n_avail;
 }
 
 static void
@@ -823,7 +886,7 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
 		VLIB_BUFFER_MAX_NUMA_NODES);
 
   /* *INDENT-OFF* */
-  clib_bitmap_foreach (numa_node, bmp,
+  clib_bitmap_foreach (numa_node, bmp)
     {
       u8 *index = bm->default_buffer_pool_index_for_numa + numa_node;
       index[0] = ~0;
@@ -836,7 +899,7 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
 
       if (first_valid_buffer_pool_index == 0xff)
         first_valid_buffer_pool_index = index[0];
-    });
+    }
   /* *INDENT-ON* */
 
   if (first_valid_buffer_pool_index == (u8) ~ 0)
@@ -846,12 +909,12 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
     }
 
   /* *INDENT-OFF* */
-  clib_bitmap_foreach (numa_node, bmp,
+  clib_bitmap_foreach (numa_node, bmp)
     {
       if (bm->default_buffer_pool_index_for_numa[numa_node]  == (u8) ~0)
 	bm->default_buffer_pool_index_for_numa[numa_node] =
 	  first_valid_buffer_pool_index;
-    });
+    }
   /* *INDENT-ON* */
 
   vec_foreach (bp, bm->buffer_pools)
@@ -890,11 +953,15 @@ vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input)
   vlib_buffer_main_alloc (vm);
 
   bm = vm->buffer_main;
+  bm->log2_page_size = CLIB_MEM_PAGE_SZ_UNKNOWN;
 
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {
       if (unformat (input, "buffers-per-numa %u", &bm->buffers_per_numa))
 	;
+      else if (unformat (input, "page-size %U", unformat_log2_page_size,
+			 &bm->log2_page_size))
+	;
       else if (unformat (input, "default data-size %u",
 			 &bm->default_data_size))
 	;
@@ -908,6 +975,24 @@ vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input)
 
 VLIB_EARLY_CONFIG_FUNCTION (vlib_buffers_configure, "buffers");
 
+#if VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0
+u32
+vlib_buffer_alloc_may_fail (vlib_main_t * vm, u32 n_buffers)
+{
+  f64 r;
+
+  r = random_f64 (&vm->buffer_alloc_success_seed);
+
+  /* Fail this request? */
+  if (r > vm->buffer_alloc_success_rate)
+    n_buffers--;
+  /* 5% chance of returning nothing at all */
+  if (r > vm->buffer_alloc_success_rate && r > 0.95)
+    n_buffers = 0;
+
+  return n_buffers;
+}
+#endif
 
 /** @endcond */
 /*