Minimize bihash memory consumption

[vpp.git] / src / vppinfra / bihash_template.h

diff --git a/src/vppinfra/bihash_template.h b/src/vppinfra/bihash_template.h
index feb6fb6..4e5d995 100644 (file)
--- a/src/vppinfra/bihash_template.h
+++ b/src/vppinfra/bihash_template.h
@@ -61,11 +61,14 @@ typedef struct
       u32 offset;
       u8 linear_search;
       u8 log2_pages;
-      u16 cache_lru;
+      i16 refcnt;
     };
     u64 as_u64;
   };
+#if BIHASH_KVP_CACHE_SIZE > 0
+  u16 cache_lru;
     BVT (clib_bihash_kv) cache[BIHASH_KVP_CACHE_SIZE];
+#endif
 } BVT (clib_bihash_bucket);
 
 typedef struct
@@ -80,7 +83,6 @@ typedef struct
 
   u32 nbuckets;
   u32 log2_nbuckets;
-  u32 linear_buckets;
   u8 *name;
 
   u64 cache_hits;
@@ -89,19 +91,22 @@ typedef struct
     BVT (clib_bihash_value) ** freelists;
   void *mheap;
 
+  /**
+    * A custom format function to print the Key and Value of bihash_key instead of default hexdump
+    */
+  format_function_t *fmt_fn;
+
 } BVT (clib_bihash);
 
 
 static inline void
 BV (clib_bihash_update_lru) (BVT (clib_bihash_bucket) * b, u8 slot)
 {
+#if BIHASH_KVP_CACHE_SIZE > 1
   u16 value, tmp, mask;
   u8 found_lru_pos;
   u16 save_hi;
 
-  if (BIHASH_KVP_CACHE_SIZE < 2)
-    return;
-
   ASSERT (slot < BIHASH_KVP_CACHE_SIZE);
 
   /* First, find the slot in cache_lru */
@@ -147,6 +152,7 @@ BV (clib_bihash_update_lru) (BVT (clib_bihash_bucket) * b, u8 slot)
   value = save_hi | (tmp << 3) | slot;
 
   b->cache_lru = value;
+#endif
 }
 
 void
@@ -155,11 +161,16 @@ BV (clib_bihash_update_lru_not_inline) (BVT (clib_bihash_bucket) * b,
 
 static inline u8 BV (clib_bihash_get_lru) (BVT (clib_bihash_bucket) * b)
 {
+#if BIHASH_KVP_CACHE_SIZE > 0
   return (b->cache_lru >> (3 * (BIHASH_KVP_CACHE_SIZE - 1))) & 7;
+#else
+  return 0;
+#endif
 }
 
 static inline void BV (clib_bihash_reset_cache) (BVT (clib_bihash_bucket) * b)
 {
+#if BIHASH_KVP_CACHE_SIZE > 0
   u16 initial_lru_value;
 
   memset (b->cache, 0xff, sizeof (b->cache));
@@ -180,16 +191,34 @@ static inline void BV (clib_bihash_reset_cache) (BVT (clib_bihash_bucket) * b)
     initial_lru_value = (0 << 12) | (1 << 9) | (2 << 6) | (3 << 3) | (4 << 0);
 
   b->cache_lru = initial_lru_value;
+#endif
+}
+
+static inline int BV (clib_bihash_lock_bucket) (BVT (clib_bihash_bucket) * b)
+{
+#if BIHASH_KVP_CACHE_SIZE > 0
+  u16 cache_lru_bit;
+  u16 rv;
+
+  cache_lru_bit = 1 << 15;
+
+  rv = __sync_fetch_and_or (&b->cache_lru, cache_lru_bit);
+  /* Was already locked? */
+  if (rv & (1 << 15))
+    return 0;
+#endif
+  return 1;
 }
 
-static inline void BV (clib_bihash_cache_enable_disable)
-  (BVT (clib_bihash_bucket) * b, u8 enable)
+static inline void BV (clib_bihash_unlock_bucket)
+  (BVT (clib_bihash_bucket) * b)
 {
-  BVT (clib_bihash_bucket) tmp_b;
-  tmp_b.as_u64 = b->as_u64;
-  tmp_b.cache_lru &= 0x7FFF;
-  tmp_b.cache_lru |= enable << 15;
-  b->as_u64 = tmp_b.as_u64;
+#if BIHASH_KVP_CACHE_SIZE > 0
+  u16 cache_lru;
+
+  cache_lru = b->cache_lru & ~(1 << 15);
+  b->cache_lru = cache_lru;
+#endif
 }
 
 static inline void *BV (clib_bihash_get_value) (BVT (clib_bihash) * h,
@@ -216,6 +245,9 @@ static inline uword BV (clib_bihash_get_offset) (BVT (clib_bihash) * h,
 void BV (clib_bihash_init)
   (BVT (clib_bihash) * h, char *name, u32 nbuckets, uword memory_size);
 
+void BV (clib_bihash_set_kvp_format_fn) (BVT (clib_bihash) * h,
+                                        format_function_t * fmt_fn);
+
 void BV (clib_bihash_free) (BVT (clib_bihash) * h);
 
 int BV (clib_bihash_add_del) (BVT (clib_bihash) * h,
@@ -238,7 +270,9 @@ static inline int BV (clib_bihash_search_inline)
   u32 bucket_index;
   BVT (clib_bihash_value) * v;
   BVT (clib_bihash_bucket) * b;
+#if BIHASH_KVP_CACHE_SIZE > 0
   BVT (clib_bihash_kv) * kvp;
+#endif
   int i, limit;
 
   hash = BV (clib_bihash_hash) (key_result);
@@ -249,8 +283,9 @@ static inline int BV (clib_bihash_search_inline)
   if (b->offset == 0)
     return -1;
 
-  /* Check the cache, if currently enabled */
-  if (PREDICT_TRUE (b->cache_lru & (1 << 15)))
+#if BIHASH_KVP_CACHE_SIZE > 0
+  /* Check the cache, if not currently locked */
+  if (PREDICT_TRUE ((b->cache_lru & (1 << 15)) == 0))
     {
       limit = BIHASH_KVP_CACHE_SIZE;
       kvp = b->cache;
@@ -264,6 +299,7 @@ static inline int BV (clib_bihash_search_inline)
            }
        }
     }
+#endif
 
   hash >>= h->log2_nbuckets;
 
@@ -279,20 +315,22 @@ static inline int BV (clib_bihash_search_inline)
     {
       if (BV (clib_bihash_key_compare) (v->kvp[i].key, key_result->key))
        {
-         u8 cache_slot;
          *key_result = v->kvp[i];
 
-         /* Shut off the cache */
-         BV (clib_bihash_cache_enable_disable) (b, 0);
-         CLIB_MEMORY_BARRIER ();
-
-         cache_slot = BV (clib_bihash_get_lru) (b);
-         b->cache[cache_slot] = v->kvp[i];
-         BV (clib_bihash_update_lru) (b, cache_slot);
+#if BIHASH_KVP_CACHE_SIZE > 0
+         u8 cache_slot;
+         /* Try to lock the bucket */
+         if (BV (clib_bihash_lock_bucket) (b))
+           {
+             cache_slot = BV (clib_bihash_get_lru) (b);
+             b->cache[cache_slot] = v->kvp[i];
+             BV (clib_bihash_update_lru) (b, cache_slot);
 
-         /* Reenable the cache */
-         BV (clib_bihash_cache_enable_disable) (b, 1);
-         h->cache_misses++;
+             /* Unlock the bucket */
+             BV (clib_bihash_unlock_bucket) (b);
+             h->cache_misses++;
+           }
+#endif
          return 0;
        }
     }
@@ -307,7 +345,9 @@ static inline int BV (clib_bihash_search_inline_2)
   u32 bucket_index;
   BVT (clib_bihash_value) * v;
   BVT (clib_bihash_bucket) * b;
+#if BIHASH_KVP_CACHE_SIZE > 0
   BVT (clib_bihash_kv) * kvp;
+#endif
   int i, limit;
 
   ASSERT (valuep);
@@ -320,8 +360,9 @@ static inline int BV (clib_bihash_search_inline_2)
   if (b->offset == 0)
     return -1;
 
-  /* Check the cache, if currently enabled */
-  if (PREDICT_TRUE (b->cache_lru & (1 << 15)))
+  /* Check the cache, if currently unlocked */
+#if BIHASH_KVP_CACHE_SIZE > 0
+  if (PREDICT_TRUE ((b->cache_lru & (1 << 15)) == 0))
     {
       limit = BIHASH_KVP_CACHE_SIZE;
       kvp = b->cache;
@@ -335,6 +376,7 @@ static inline int BV (clib_bihash_search_inline_2)
            }
        }
     }
+#endif
 
   hash >>= h->log2_nbuckets;
   v = BV (clib_bihash_get_value) (h, b->offset);
@@ -349,20 +391,23 @@ static inline int BV (clib_bihash_search_inline_2)
     {
       if (BV (clib_bihash_key_compare) (v->kvp[i].key, search_key->key))
        {
-         u8 cache_slot;
          *valuep = v->kvp[i];
 
-         /* Shut off the cache */
-         BV (clib_bihash_cache_enable_disable) (b, 0);
-         CLIB_MEMORY_BARRIER ();
+#if BIHASH_KVP_CACHE_SIZE > 0
+         u8 cache_slot;
 
-         cache_slot = BV (clib_bihash_get_lru) (b);
-         b->cache[cache_slot] = v->kvp[i];
-         BV (clib_bihash_update_lru) (b, cache_slot);
+         /* Try to lock the bucket */
+         if (BV (clib_bihash_lock_bucket) (b))
+           {
+             cache_slot = BV (clib_bihash_get_lru) (b);
+             b->cache[cache_slot] = v->kvp[i];
+             BV (clib_bihash_update_lru) (b, cache_slot);
 
-         /* Reenable the cache */
-         BV (clib_bihash_cache_enable_disable) (b, 1);
-         h->cache_misses++;
+             /* Reenable the cache */
+             BV (clib_bihash_unlock_bucket) (b);
+             h->cache_misses++;
+           }
+#endif
          return 0;
        }
     }