X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;ds=sidebyside;f=src%2Fvppinfra%2Fbihash_template.c;h=a10dee025bf11cb5b3d593fc6d7988a116144904;hb=2782630c00efa4349567b908b9534d963b6f67af;hp=2571c47e64c306e86eed9c9e09a61690932fffcc;hpb=a713254d689deee1718bb515c3961bf4ed435504;p=vpp.git

diff --git a/src/vppinfra/bihash_template.c b/src/vppinfra/bihash_template.c
index 2571c47e64c..a10dee025bf 100644
--- a/src/vppinfra/bihash_template.c
+++ b/src/vppinfra/bihash_template.c
@@ -196,6 +196,7 @@ void BV (clib_bihash_set_kvp_format_fn) (BVT (clib_bihash) * h,
 void BV (clib_bihash_free) (BVT (clib_bihash) * h)
 {
   vec_free (h->working_copies);
+  vec_free (h->working_copy_lengths);
 #if BIHASH_32_64_SVM == 0
   vec_free (h->freelists);
 #else
@@ -203,7 +204,7 @@ void BV (clib_bihash_free) (BVT (clib_bihash) * h)
     (void) close (h->memfd);
 #endif
   clib_mem_vm_free ((void *) (uword) (alloc_arena (h)), alloc_arena_size (h));
-  memset (h, 0, sizeof (*h));
+  clib_memset (h, 0, sizeof (*h));
 }
 
 static
@@ -234,7 +235,7 @@ initialize:
    * if we replace (1<<log2_pages) with vec_len(rv).
    * No clue.
    */
-  memset (rv, 0xff, sizeof (*rv) * (1 << log2_pages));
+  clib_memset (rv, 0xff, sizeof (*rv) * (1 << log2_pages));
   return rv;
 }
 
@@ -247,7 +248,7 @@ BV (value_free) (BVT (clib_bihash) * h, BVT (clib_bihash_value) * v,
   ASSERT (vec_len (h->freelists) > log2_pages);
 
   if (CLIB_DEBUG > 0)
-    memset (v, 0xFE, sizeof (*v) * (1 << log2_pages));
+    clib_memset (v, 0xFE, sizeof (*v) * (1 << log2_pages));
 
   v->next_free_as_u64 = (u64) h->freelists[log2_pages];
   h->freelists[log2_pages] = (u64) BV (clib_bihash_get_offset) (h, v);
@@ -291,11 +292,14 @@ BV (make_working_copy) (BVT (clib_bihash) * h, BVT (clib_bihash_bucket) * b)
 	(h, sizeof (working_copy[0]) * (1 << b->log2_pages));
       h->working_copy_lengths[thread_index] = b->log2_pages;
       h->working_copies[thread_index] = working_copy;
+
+      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_working_copy_lost,
+				       1ULL << b->log2_pages);
     }
 
   v = BV (clib_bihash_get_value) (h, b->offset);
 
-  clib_memcpy (working_copy, v, sizeof (*v) * (1 << b->log2_pages));
+  clib_memcpy_fast (working_copy, v, sizeof (*v) * (1 << b->log2_pages));
   working_bucket.as_u64 = b->as_u64;
   working_bucket.offset = BV (clib_bihash_get_offset) (h, working_copy);
   CLIB_MEMORY_BARRIER ();
@@ -338,8 +342,8 @@ BV (split_and_rehash)
 	  /* Empty slot */
 	  if (BV (clib_bihash_is_free) (&(new_v->kvp[j])))
 	    {
-	      clib_memcpy (&(new_v->kvp[j]), &(old_values->kvp[i]),
-			   sizeof (new_v->kvp[j]));
+	      clib_memcpy_fast (&(new_v->kvp[j]), &(old_values->kvp[i]),
+				sizeof (new_v->kvp[j]));
 	      goto doublebreak;
 	    }
 	}
@@ -383,8 +387,8 @@ BV (split_and_rehash_linear)
 	  if (BV (clib_bihash_is_free) (&(new_values->kvp[j])))
 	    {
 	      /* Copy the old value and move along */
-	      clib_memcpy (&(new_values->kvp[j]), &(old_values->kvp[i]),
-			   sizeof (new_values->kvp[j]));
+	      clib_memcpy_fast (&(new_values->kvp[j]), &(old_values->kvp[i]),
+				sizeof (new_values->kvp[j]));
 	      j++;
 	      goto doublebreak;
 	    }
@@ -441,8 +445,9 @@ static inline int BV (clib_bihash_add_del_inline)
       tmp_b.refcnt = 1;
       CLIB_MEMORY_BARRIER ();
 
-      b->as_u64 = tmp_b.as_u64;
-      BV (clib_bihash_unlock_bucket) (b);
+      b->as_u64 = tmp_b.as_u64;	/* unlocks the bucket */
+      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_alloc_add, 1);
+
       return (0);
     }
 
@@ -469,11 +474,12 @@ static inline int BV (clib_bihash_add_del_inline)
        */
       for (i = 0; i < limit; i++)
 	{
-	  if (!memcmp (&(v->kvp[i]), &add_v->key, sizeof (add_v->key)))
+	  if (BV (clib_bihash_key_compare) (v->kvp[i].key, add_v->key))
 	    {
 	      CLIB_MEMORY_BARRIER ();	/* Add a delay */
-	      clib_memcpy (&(v->kvp[i]), add_v, sizeof (*add_v));
+	      clib_memcpy_fast (&(v->kvp[i]), add_v, sizeof (*add_v));
 	      BV (clib_bihash_unlock_bucket) (b);
+	      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_replace, 1);
 	      return (0);
 	    }
 	}
@@ -488,13 +494,15 @@ static inline int BV (clib_bihash_add_del_inline)
 	       * Copy the value first, so that if a reader manages
 	       * to match the new key, the value will be right...
 	       */
-	      clib_memcpy (&(v->kvp[i].value),
-			   &add_v->value, sizeof (add_v->value));
+	      clib_memcpy_fast (&(v->kvp[i].value),
+				&add_v->value, sizeof (add_v->value));
 	      CLIB_MEMORY_BARRIER ();	/* Make sure the value has settled */
-	      clib_memcpy (&(v->kvp[i]), &add_v->key, sizeof (add_v->key));
+	      clib_memcpy_fast (&(v->kvp[i]), &add_v->key,
+				sizeof (add_v->key));
 	      b->refcnt++;
 	      ASSERT (b->refcnt > 0);
 	      BV (clib_bihash_unlock_bucket) (b);
+	      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_add, 1);
 	      return (0);
 	    }
 	}
@@ -506,8 +514,9 @@ static inline int BV (clib_bihash_add_del_inline)
 	      if (is_stale_cb (&(v->kvp[i]), arg))
 		{
 		  CLIB_MEMORY_BARRIER ();
-		  clib_memcpy (&(v->kvp[i]), add_v, sizeof (*add_v));
+		  clib_memcpy_fast (&(v->kvp[i]), add_v, sizeof (*add_v));
 		  BV (clib_bihash_unlock_bucket) (b);
+		  BV (clib_bihash_increment_stat) (h, BIHASH_STAT_replace, 1);
 		  return (0);
 		}
 	    }
@@ -519,14 +528,15 @@ static inline int BV (clib_bihash_add_del_inline)
       for (i = 0; i < limit; i++)
 	{
 	  /* Found the key? Kill it... */
-	  if (!memcmp (&(v->kvp[i]), &add_v->key, sizeof (add_v->key)))
+	  if (BV (clib_bihash_key_compare) (v->kvp[i].key, add_v->key))
 	    {
-	      memset (&(v->kvp[i]), 0xff, sizeof (*(add_v)));
+	      clib_memset (&(v->kvp[i]), 0xff, sizeof (*(add_v)));
 	      /* Is the bucket empty? */
 	      if (PREDICT_TRUE (b->refcnt > 1))
 		{
 		  b->refcnt--;
 		  BV (clib_bihash_unlock_bucket) (b);
+		  BV (clib_bihash_increment_stat) (h, BIHASH_STAT_del, 1);
 		  return (0);
 		}
 	      else		/* yes, free it */
@@ -544,6 +554,8 @@ static inline int BV (clib_bihash_add_del_inline)
 		  v = BV (clib_bihash_get_value) (h, tmp_b.offset);
 		  BV (value_free) (h, v, tmp_b.log2_pages);
 		  BV (clib_bihash_alloc_unlock) (h);
+		  BV (clib_bihash_increment_stat) (h, BIHASH_STAT_del_free,
+						   1);
 		  return (0);
 		}
 	    }
@@ -562,9 +574,12 @@ static inline int BV (clib_bihash_add_del_inline)
   old_log2_pages = h->saved_bucket.log2_pages;
   new_log2_pages = old_log2_pages + 1;
   mark_bucket_linear = 0;
+  BV (clib_bihash_increment_stat) (h, BIHASH_STAT_split_add, 1);
+  BV (clib_bihash_increment_stat) (h, BIHASH_STAT_splits, old_log2_pages);
 
   working_copy = h->working_copies[thread_index];
   resplit_once = 0;
+  BV (clib_bihash_increment_stat) (h, BIHASH_STAT_splits, 1);
 
   new_v = BV (split_and_rehash) (h, working_copy, old_log2_pages,
 				 new_log2_pages);
@@ -585,7 +600,11 @@ static inline int BV (clib_bihash_add_del_inline)
 	    BV (split_and_rehash_linear) (h, working_copy, old_log2_pages,
 					  new_log2_pages);
 	  mark_bucket_linear = 1;
+	  BV (clib_bihash_increment_stat) (h, BIHASH_STAT_linear, 1);
 	}
+      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_resplit, 1);
+      BV (clib_bihash_increment_stat) (h, BIHASH_STAT_splits,
+				       old_log2_pages + 1);
     }
 
   /* Try to add the new entry */
@@ -602,7 +621,7 @@ static inline int BV (clib_bihash_add_del_inline)
     {
       if (BV (clib_bihash_is_free) (&(new_v->kvp[i])))
 	{
-	  clib_memcpy (&(new_v->kvp[i]), add_v, sizeof (*add_v));
+	  clib_memcpy_fast (&(new_v->kvp[i]), add_v, sizeof (*add_v));
 	  goto expand_ok;
 	}
     }
@@ -627,6 +646,9 @@ expand_ok:
   tmp_b.lock = 0;
   CLIB_MEMORY_BARRIER ();
   b->as_u64 = tmp_b.as_u64;
+  /* free the old bucket */
+  v = BV (clib_bihash_get_value) (h, h->saved_bucket.offset);
+  BV (value_free) (h, v, h->saved_bucket.log2_pages);
   BV (clib_bihash_alloc_unlock) (h);
   return (0);
 }
@@ -743,7 +765,7 @@ u8 *BV (format_bihash) (u8 * s, va_list * args)
 		    {
 		      s = format (s, "    %d: %U\n",
 				  j * BIHASH_KVP_PER_PAGE + k,
-				  h->fmt_fn, &(v->kvp[k]));
+				  h->fmt_fn, &(v->kvp[k]), verbose);
 		    }
 		  else
 		    {