vppinfra: numa vector placement support

[vpp.git] / src / vppinfra / bihash_template.c

diff --git a/src/vppinfra/bihash_template.c b/src/vppinfra/bihash_template.c
index cd75a7a..2b37842 100644 (file)
--- a/src/vppinfra/bihash_template.c
+++ b/src/vppinfra/bihash_template.c
@@ -26,13 +26,13 @@ static inline void *BV (alloc_aligned) (BVT (clib_bihash) * h, uword nbytes)
   rv = alloc_arena_next (h);
   alloc_arena_next (h) += nbytes;
 
-  if (rv >= alloc_arena_size (h))
+  if (alloc_arena_next (h) > alloc_arena_size (h))
     os_out_of_memory ();
 
   return (void *) (uword) (rv + alloc_arena (h));
 }
 
-void BV (clib_bihash_instantiate) (BVT (clib_bihash) * h)
+static void BV (clib_bihash_instantiate) (BVT (clib_bihash) * h)
 {
   uword bucket_size;
 
@@ -42,22 +42,25 @@ void BV (clib_bihash_instantiate) (BVT (clib_bihash) * h)
 
   bucket_size = h->nbuckets * sizeof (h->buckets[0]);
   h->buckets = BV (alloc_aligned) (h, bucket_size);
-
-  h->alloc_lock = BV (alloc_aligned) (h, CLIB_CACHE_LINE_BYTES);
-  h->alloc_lock[0] = 0;
+  CLIB_MEMORY_BARRIER ();
+  h->instantiated = 1;
 }
 
-void BV (clib_bihash_init)
-  (BVT (clib_bihash) * h, char *name, u32 nbuckets, uword memory_size)
+void BV (clib_bihash_init2) (BVT (clib_bihash_init2_args) * a)
 {
   int i;
   void *oldheap;
-  nbuckets = 1 << (max_log2 (nbuckets));
+  BVT (clib_bihash) * h = a->h;
+
+  a->nbuckets = 1 << (max_log2 (a->nbuckets));
+
+  h->name = (u8 *) a->name;
+  h->nbuckets = a->nbuckets;
+  h->log2_nbuckets = max_log2 (a->nbuckets);
+  h->memory_size = a->memory_size;
+  h->instantiated = 0;
+  h->fmt_fn = a->fmt_fn;
 
-  h->name = (u8 *) name;
-  h->nbuckets = nbuckets;
-  h->log2_nbuckets = max_log2 (nbuckets);
-  h->memory_size = memory_size;
   alloc_arena (h) = 0;
 
   /*
@@ -66,22 +69,48 @@ void BV (clib_bihash_init)
    * If someone starts complaining that's not enough, we can shift
    * the offset by CLIB_LOG2_CACHE_LINE_BYTES...
    */
-  ASSERT (memory_size < (1ULL << BIHASH_BUCKET_OFFSET_BITS));
-  h->fmt_fn = NULL;
+  ASSERT (h->memory_size < (1ULL << BIHASH_BUCKET_OFFSET_BITS));
 
   /* Add this hash table to the list */
-  for (i = 0; i < vec_len (clib_all_bihashes); i++)
-    if (clib_all_bihashes[i] == h)
-      return;
+  if (a->dont_add_to_all_bihash_list == 0)
+    {
+      for (i = 0; i < vec_len (clib_all_bihashes); i++)
+       if (clib_all_bihashes[i] == h)
+         goto do_lock;
+      oldheap = clib_all_bihash_set_heap ();
+      vec_add1 (clib_all_bihashes, (void *) h);
+      clib_mem_set_heap (oldheap);
+    }
 
-  /* Unfortunately, the heap push/pop is required.... */
-  oldheap = clib_all_bihash_set_heap ();
-  vec_add1 (clib_all_bihashes, (void *) h);
-  clib_mem_set_heap (oldheap);
+do_lock:
+  if (h->alloc_lock)
+    clib_mem_free ((void *) h->alloc_lock);
 
-#if BIHASH_INSTANTIATE_IMMEDIATELY
-  BV (clib_bihash_instantiate) (h);
-#endif
+  /*
+   * Set up the lock now, so we can use it to make the first add
+   * thread-safe
+   */
+  h->alloc_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
+                                         CLIB_CACHE_LINE_BYTES);
+  h->alloc_lock[0] = 0;
+
+  if (a->instantiate_immediately)
+    BV (clib_bihash_instantiate) (h);
+}
+
+void BV (clib_bihash_init)
+  (BVT (clib_bihash) * h, char *name, u32 nbuckets, uword memory_size)
+{
+  BVT (clib_bihash_init2_args) _a, *a = &_a;
+
+  memset (a, 0, sizeof (*a));
+
+  a->h = h;
+  a->name = name;
+  a->nbuckets = nbuckets;
+  a->memory_size = memory_size;
+
+  BV (clib_bihash_init2) (a);
 }
 
 #if BIHASH_32_64_SVM
@@ -150,12 +179,12 @@ void BV (clib_bihash_master_init_svm)
                        sizeof (vec_header_t) +
                        BIHASH_FREELIST_LENGTH * sizeof (u64));
   freelist_vh->len = BIHASH_FREELIST_LENGTH;
-  freelist_vh->dlmalloc_header_offset = 0xDEADBEEF;
   h->sh->freelists_as_u64 =
     (u64) BV (clib_bihash_get_offset) (h, freelist_vh->vector_data);
   h->freelists = (void *) (freelist_vh->vector_data);
 
   h->fmt_fn = NULL;
+  h->instantiated = 1;
 }
 
 void BV (clib_bihash_slave_init_svm)
@@ -216,9 +245,10 @@ void BV (clib_bihash_free) (BVT (clib_bihash) * h)
 {
   int i;
 
-  if (PREDICT_FALSE (alloc_arena (h) == 0))
+  if (PREDICT_FALSE (h->instantiated == 0))
     goto never_initialized;
 
+  h->instantiated = 0;
   vec_free (h->working_copies);
   vec_free (h->working_copy_lengths);
 #if BIHASH_32_64_SVM == 0
@@ -239,7 +269,7 @@ never_initialized:
        }
     }
   clib_warning ("Couldn't find hash table %llx on clib_all_bihashes...",
-               (u64) h);
+               (u64) (uword) h);
 }
 
 static
@@ -452,12 +482,19 @@ static inline int BV (clib_bihash_add_del_inline)
   int mark_bucket_linear;
   int resplit_once;
 
-  /* Create the table (is_add=1), or flunk the request now (is_add=0) */
-  if (PREDICT_FALSE (alloc_arena (h) == 0))
+  /*
+   * Create the table (is_add=1,2), or flunk the request now (is_add=0)
+   * Use the alloc_lock to protect the instantiate operation.
+   */
+  if (PREDICT_FALSE (h->instantiated == 0))
     {
       if (is_add == 0)
        return (-1);
-      BV (clib_bihash_instantiate) (h);
+
+      BV (clib_bihash_alloc_lock) (h);
+      if (h->instantiated == 0)
+       BV (clib_bihash_instantiate) (h);
+      BV (clib_bihash_alloc_unlock) (h);
     }
 
   hash = BV (clib_bihash_hash) (add_v);
@@ -869,12 +906,12 @@ u8 *BV (format_bihash) (u8 * s, va_list * args)
 }
 
 void BV (clib_bihash_foreach_key_value_pair)
-  (BVT (clib_bihash) * h, void *callback, void *arg)
+  (BVT (clib_bihash) * h,
+   BV (clib_bihash_foreach_key_value_pair_cb) cb, void *arg)
 {
   int i, j, k;
   BVT (clib_bihash_bucket) * b;
   BVT (clib_bihash_value) * v;
-  void (*fp) (BVT (clib_bihash_kv) *, void *) = callback;
 
   if (PREDICT_FALSE (alloc_arena (h) == 0))
     return;
@@ -893,7 +930,8 @@ void BV (clib_bihash_foreach_key_value_pair)
              if (BV (clib_bihash_is_free) (&v->kvp[k]))
                continue;
 
-             (*fp) (&v->kvp[k], arg);
+             if (BIHASH_WALK_STOP == cb (&v->kvp[k], arg))
+               return;
              /*
               * In case the callback deletes the last entry in the bucket...
               */