X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fmheap.c;h=5b71873b9ed255f9af538a8e63d335a14c108b2e;hb=6a5adc369591fcac2447e9809deaa22f56b53911;hp=d4010ceb2978c91e2f765613ebcd02b92ce2ce65;hpb=f55f9b851f59264d737d92c6277a87588c565d24;p=vpp.git

diff --git a/src/vppinfra/mheap.c b/src/vppinfra/mheap.c
index d4010ceb297..5b71873b9ed 100644
--- a/src/vppinfra/mheap.c
+++ b/src/vppinfra/mheap.c
@@ -311,7 +311,7 @@ mheap_small_object_cache_mask (mheap_small_object_cache_t * c, uword bin)
 
   ASSERT (bin < 256);
 
-#define _(i) ((uword) u8x16_compare_byte_mask (u8x16_is_equal (b, c->bins.as_u8x16[i])) << (uword) ((i)*16))
+#define _(i) ((uword) u8x16_compare_byte_mask ((b == c->bins.as_u8x16[i])) << (uword) ((i)*16))
   mask = _(0) | _(1);
   if (BITS (uword) > 32)
     mask |= _(2) | _(3);
@@ -549,23 +549,17 @@ mheap_get_search_free_list (void *v,
 	non_empty_bin_mask &= ~pow2_mask (bin % BITS (uword));
 
       /* Search each occupied free bin which is large enough. */
-      foreach_set_bit (bi, non_empty_bin_mask, (
-						 {
-						 uword r =
-						 mheap_get_search_free_bin (v,
-									    bi
-									    +
-									    i
-									    *
-									    BITS
-									    (uword),
-									    n_user_bytes_arg,
-									    align,
-									    align_offset);
-						 if (r !=
-						     MHEAP_GROUNDED) return
-						 r;}
-		       ));
+      /* *INDENT-OFF* */
+      foreach_set_bit (bi, non_empty_bin_mask,
+      ({
+        uword r =
+          mheap_get_search_free_bin (v, bi + i * BITS (uword),
+                                     n_user_bytes_arg,
+                                     align,
+                                     align_offset);
+        if (r != MHEAP_GROUNDED) return r;
+      }));
+      /* *INDENT-ON* */
     }
 
   return MHEAP_GROUNDED;
@@ -669,12 +663,28 @@ mheap_get_aligned (void *v,
       return v;
     }
 
-  /* Round requested size. */
+  /*
+   * Round requested size.
+   *
+   * Step 1: round up to the minimum object size.
+   * Step 2: round up to a multiple of the user data size (e.g. 4)
+   * Step 3: if non-trivial alignment requested, round up
+   *         so that the object precisely fills a chunk
+   *         as big as the alignment request.
+   *
+   * Step 3 prevents the code from going into "bin search hyperspace":
+   * looking at a huge number of fractional remainder chunks, none of which
+   * will satisfy the alignment constraint. This fixes an allocator
+   * performance issue when one requests a large number of 16 byte objects
+   * aligned to 64 bytes, to name one variation on the theme.
+   */
   n_user_data_bytes = clib_max (n_user_data_bytes, MHEAP_MIN_USER_DATA_BYTES);
   n_user_data_bytes =
     round_pow2 (n_user_data_bytes,
 		STRUCT_SIZE_OF (mheap_elt_t, user_data[0]));
-
+  if (align > MHEAP_ELT_OVERHEAD_BYTES)
+    n_user_data_bytes = clib_max (n_user_data_bytes,
+				  align - MHEAP_ELT_OVERHEAD_BYTES);
   if (!v)
     v = mheap_alloc (0, 64 << 20);
 
@@ -964,6 +974,29 @@ mheap_alloc (void *memory, uword size)
   return mheap_alloc_with_flags (memory, size, flags);
 }
 
+void *
+mheap_alloc_with_lock (void *memory, uword size, int locked)
+{
+  uword flags = 0;
+  void *rv;
+
+  if (memory != 0)
+    flags |= MHEAP_FLAG_DISABLE_VM;
+
+#ifdef CLIB_HAVE_VEC128
+  flags |= MHEAP_FLAG_SMALL_OBJECT_CACHE;
+#endif
+
+  rv = mheap_alloc_with_flags (memory, size, flags);
+
+  if (rv && locked)
+    {
+      mheap_t *h = mheap_header (rv);
+      h->flags |= MHEAP_FLAG_THREAD_SAFE;
+    }
+  return rv;
+}
+
 void *
 _mheap_free (void *v)
 {
@@ -1126,7 +1159,7 @@ format_mheap_stats (u8 * s, va_list * va)
 {
   mheap_t *h = va_arg (*va, mheap_t *);
   mheap_stats_t *st = &h->stats;
-  uword indent = format_get_indent (s);
+  u32 indent = format_get_indent (s);
 
   s =
     format (s,
@@ -1171,7 +1204,8 @@ format_mheap (u8 * s, va_list * va)
   int verbose = va_arg (*va, int);
 
   mheap_t *h;
-  uword i, size, indent;
+  uword i, size;
+  u32 indent;
   clib_mem_usage_t usage;
   mheap_elt_t *first_corrupt;
 
@@ -1242,7 +1276,7 @@ format_mheap (u8 * s, va_list * va)
     {
       /* Make a copy of traces since we'll be sorting them. */
       mheap_trace_t *t, *traces_copy;
-      uword indent, total_objects_traced;
+      u32 indent, total_objects_traced;
 
       traces_copy = vec_dup (h->trace_main.traces);
       qsort (traces_copy, vec_len (traces_copy), sizeof (traces_copy[0]),
@@ -1517,7 +1551,7 @@ mheap_get_trace (void *v, uword offset, uword size)
 
   if (!tm->trace_by_callers)
     tm->trace_by_callers =
-      hash_create_mem (0, sizeof (trace.callers), sizeof (uword));
+      hash_create_shmem (0, sizeof (trace.callers), sizeof (uword));
 
   p = hash_get_mem (tm->trace_by_callers, &trace.callers);
   if (p)