vppinfra: conformed spinlocks to use CLIB_PAUSE

[vpp.git] / src / vppinfra / bihash_template.c

diff --git a/src/vppinfra/bihash_template.c b/src/vppinfra/bihash_template.c
index 2840342..11e3ac4 100644 (file)
--- a/src/vppinfra/bihash_template.c
+++ b/src/vppinfra/bihash_template.c
@@ -26,23 +26,39 @@ static inline void *BV (alloc_aligned) (BVT (clib_bihash) * h, uword nbytes)
   rv = alloc_arena_next (h);
   alloc_arena_next (h) += nbytes;
 
-  if (rv >= alloc_arena_size (h))
+  if (alloc_arena_next (h) > alloc_arena_size (h))
     os_out_of_memory ();
 
   return (void *) (uword) (rv + alloc_arena (h));
 }
 
+void BV (clib_bihash_instantiate) (BVT (clib_bihash) * h)
+{
+  uword bucket_size;
+
+  alloc_arena (h) = (uword) clib_mem_vm_alloc (h->memory_size);
+  alloc_arena_next (h) = 0;
+  alloc_arena_size (h) = h->memory_size;
+
+  bucket_size = h->nbuckets * sizeof (h->buckets[0]);
+  h->buckets = BV (alloc_aligned) (h, bucket_size);
+
+  h->alloc_lock = BV (alloc_aligned) (h, CLIB_CACHE_LINE_BYTES);
+  h->alloc_lock[0] = 0;
+}
 
 void BV (clib_bihash_init)
   (BVT (clib_bihash) * h, char *name, u32 nbuckets, uword memory_size)
 {
-  uword bucket_size;
-
+  int i;
+  void *oldheap;
   nbuckets = 1 << (max_log2 (nbuckets));
 
   h->name = (u8 *) name;
   h->nbuckets = nbuckets;
   h->log2_nbuckets = max_log2 (nbuckets);
+  h->memory_size = memory_size;
+  alloc_arena (h) = 0;
 
   /*
    * Make sure the requested size is rational. The max table
@@ -51,18 +67,21 @@ void BV (clib_bihash_init)
    * the offset by CLIB_LOG2_CACHE_LINE_BYTES...
    */
   ASSERT (memory_size < (1ULL << BIHASH_BUCKET_OFFSET_BITS));
+  h->fmt_fn = NULL;
 
-  alloc_arena (h) = (uword) clib_mem_vm_alloc (memory_size);
-  alloc_arena_next (h) = 0;
-  alloc_arena_size (h) = memory_size;
-
-  bucket_size = nbuckets * sizeof (h->buckets[0]);
-  h->buckets = BV (alloc_aligned) (h, bucket_size);
+  /* Add this hash table to the list */
+  for (i = 0; i < vec_len (clib_all_bihashes); i++)
+    if (clib_all_bihashes[i] == h)
+      return;
 
-  h->alloc_lock = BV (alloc_aligned) (h, CLIB_CACHE_LINE_BYTES);
-  h->alloc_lock[0] = 0;
+  /* Unfortunately, the heap push/pop is required.... */
+  oldheap = clib_all_bihash_set_heap ();
+  vec_add1 (clib_all_bihashes, (void *) h);
+  clib_mem_set_heap (oldheap);
 
-  h->fmt_fn = NULL;
+#if BIHASH_INSTANTIATE_IMMEDIATELY
+  BV (clib_bihash_instantiate) (h);
+#endif
 }
 
 #if BIHASH_32_64_SVM
@@ -195,7 +214,13 @@ void BV (clib_bihash_set_kvp_format_fn) (BVT (clib_bihash) * h,
 
 void BV (clib_bihash_free) (BVT (clib_bihash) * h)
 {
+  int i;
+
+  if (PREDICT_FALSE (alloc_arena (h) == 0))
+    goto never_initialized;
+
   vec_free (h->working_copies);
+  vec_free (h->working_copy_lengths);
 #if BIHASH_32_64_SVM == 0
   vec_free (h->freelists);
 #else
@@ -203,7 +228,18 @@ void BV (clib_bihash_free) (BVT (clib_bihash) * h)
     (void) close (h->memfd);
 #endif
   clib_mem_vm_free ((void *) (uword) (alloc_arena (h)), alloc_arena_size (h));
+never_initialized:
   clib_memset (h, 0, sizeof (*h));
+  for (i = 0; i < vec_len (clib_all_bihashes); i++)
+    {
+      if ((void *) h == clib_all_bihashes[i])
+       {
+         vec_delete (clib_all_bihashes, 1, i);
+         return;
+       }
+    }
+  clib_warning ("Couldn't find hash table %llx on clib_all_bihashes...",
+               (u64) h);
 }
 
 static
@@ -291,6 +327,9 @@ BV (make_working_copy) (BVT (clib_bihash) * h, BVT (clib_bihash_bucket) * b)
        (h, sizeof (working_copy[0]) * (1 << b->log2_pages));
       h->working_copy_lengths[thread_index] = b->log2_pages;
       h->working_copies[thread_index] = working_copy;
+
+      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_working_copy_lost,
+                                      1ULL << b->log2_pages);
     }
 
   v = BV (clib_bihash_get_value) (h, b->offset);
@@ -413,6 +452,14 @@ static inline int BV (clib_bihash_add_del_inline)
   int mark_bucket_linear;
   int resplit_once;
 
+  /* Create the table (is_add=1), or flunk the request now (is_add=0) */
+  if (PREDICT_FALSE (alloc_arena (h) == 0))
+    {
+      if (is_add == 0)
+       return (-1);
+      BV (clib_bihash_instantiate) (h);
+    }
+
   hash = BV (clib_bihash_hash) (add_v);
 
   bucket_index = hash & (h->nbuckets - 1);
@@ -442,6 +489,8 @@ static inline int BV (clib_bihash_add_del_inline)
       CLIB_MEMORY_BARRIER ();
 
       b->as_u64 = tmp_b.as_u64;        /* unlocks the bucket */
+      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_alloc_add, 1);
+
       return (0);
     }
 
@@ -470,9 +519,17 @@ static inline int BV (clib_bihash_add_del_inline)
        {
          if (BV (clib_bihash_key_compare) (v->kvp[i].key, add_v->key))
            {
+             /* Add but do not overwrite? */
+             if (is_add == 2)
+               {
+                 BV (clib_bihash_unlock_bucket) (b);
+                 return (-2);
+               }
+
              CLIB_MEMORY_BARRIER ();   /* Add a delay */
              clib_memcpy_fast (&(v->kvp[i]), add_v, sizeof (*add_v));
              BV (clib_bihash_unlock_bucket) (b);
+             BV (clib_bihash_increment_stat) (h, BIHASH_STAT_replace, 1);
              return (0);
            }
        }
@@ -495,6 +552,7 @@ static inline int BV (clib_bihash_add_del_inline)
              b->refcnt++;
              ASSERT (b->refcnt > 0);
              BV (clib_bihash_unlock_bucket) (b);
+             BV (clib_bihash_increment_stat) (h, BIHASH_STAT_add, 1);
              return (0);
            }
        }
@@ -508,6 +566,7 @@ static inline int BV (clib_bihash_add_del_inline)
                  CLIB_MEMORY_BARRIER ();
                  clib_memcpy_fast (&(v->kvp[i]), add_v, sizeof (*add_v));
                  BV (clib_bihash_unlock_bucket) (b);
+                 BV (clib_bihash_increment_stat) (h, BIHASH_STAT_replace, 1);
                  return (0);
                }
            }
@@ -527,6 +586,7 @@ static inline int BV (clib_bihash_add_del_inline)
                {
                  b->refcnt--;
                  BV (clib_bihash_unlock_bucket) (b);
+                 BV (clib_bihash_increment_stat) (h, BIHASH_STAT_del, 1);
                  return (0);
                }
              else              /* yes, free it */
@@ -544,6 +604,8 @@ static inline int BV (clib_bihash_add_del_inline)
                  v = BV (clib_bihash_get_value) (h, tmp_b.offset);
                  BV (value_free) (h, v, tmp_b.log2_pages);
                  BV (clib_bihash_alloc_unlock) (h);
+                 BV (clib_bihash_increment_stat) (h, BIHASH_STAT_del_free,
+                                                  1);
                  return (0);
                }
            }
@@ -562,9 +624,12 @@ static inline int BV (clib_bihash_add_del_inline)
   old_log2_pages = h->saved_bucket.log2_pages;
   new_log2_pages = old_log2_pages + 1;
   mark_bucket_linear = 0;
+  BV (clib_bihash_increment_stat) (h, BIHASH_STAT_split_add, 1);
+  BV (clib_bihash_increment_stat) (h, BIHASH_STAT_splits, old_log2_pages);
 
   working_copy = h->working_copies[thread_index];
   resplit_once = 0;
+  BV (clib_bihash_increment_stat) (h, BIHASH_STAT_splits, 1);
 
   new_v = BV (split_and_rehash) (h, working_copy, old_log2_pages,
                                 new_log2_pages);
@@ -585,7 +650,11 @@ static inline int BV (clib_bihash_add_del_inline)
            BV (split_and_rehash_linear) (h, working_copy, old_log2_pages,
                                          new_log2_pages);
          mark_bucket_linear = 1;
+         BV (clib_bihash_increment_stat) (h, BIHASH_STAT_linear, 1);
        }
+      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_resplit, 1);
+      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_splits,
+                                      old_log2_pages + 1);
     }
 
   /* Try to add the new entry */
@@ -659,6 +728,9 @@ int BV (clib_bihash_search)
 
   ASSERT (valuep);
 
+  if (PREDICT_FALSE (alloc_arena (h) == 0))
+    return -1;
+
   hash = BV (clib_bihash_hash) (search_key);
 
   bucket_index = hash & (h->nbuckets - 1);
@@ -707,6 +779,9 @@ u8 *BV (format_bihash) (u8 * s, va_list * args)
 
   s = format (s, "Hash table %s\n", h->name ? h->name : (u8 *) "(unnamed)");
 
+  if (PREDICT_FALSE (alloc_arena (h) == 0))
+    return format (s, "[empty, uninitialized]");
+
   for (i = 0; i < h->nbuckets; i++)
     {
       b = &h->buckets[i];
@@ -746,7 +821,7 @@ u8 *BV (format_bihash) (u8 * s, va_list * args)
                    {
                      s = format (s, "    %d: %U\n",
                                  j * BIHASH_KVP_PER_PAGE + k,
-                                 h->fmt_fn, &(v->kvp[k]));
+                                 h->fmt_fn, &(v->kvp[k]), verbose);
                    }
                  else
                    {
@@ -801,6 +876,9 @@ void BV (clib_bihash_foreach_key_value_pair)
   BVT (clib_bihash_value) * v;
   void (*fp) (BVT (clib_bihash_kv) *, void *) = callback;
 
+  if (PREDICT_FALSE (alloc_arena (h) == 0))
+    return;
+
   for (i = 0; i < h->nbuckets; i++)
     {
       b = &h->buckets[i];