X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fbihash_template.h;h=98dcf14673f8db97d0978cde5021329a9b6c8d78;hb=8c8acc0;hp=cfb8ceac69eb52e567809cea7d46a5d2374365be;hpb=828d27ea0edb97280a0164041a286973fa74b5a2;p=vpp.git

diff --git a/src/vppinfra/bihash_template.h b/src/vppinfra/bihash_template.h
index cfb8ceac69e..98dcf14673f 100644
--- a/src/vppinfra/bihash_template.h
+++ b/src/vppinfra/bihash_template.h
@@ -33,6 +33,17 @@
 #error BIHASH_TYPE not defined
 #endif
 
+#ifdef BIHASH_32_64_SVM
+#undef HAVE_MEMFD_CREATE
+#include <vppinfra/linux/syscall.h>
+#include <fcntl.h>
+#define F_LINUX_SPECIFIC_BASE 1024
+#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
+#define F_SEAL_SHRINK (2)
+/* Max page size 2**16 due to refcount width  */
+#define BIHASH_FREELIST_LENGTH 17
+#endif
+
 #define _bv(a,b) a##b
 #define __bv(a,b) _bv(a,b)
 #define BV(a) __bv(a,BIHASH_TYPE)
@@ -41,12 +52,22 @@
 #define __bvt(a,b) _bvt(a,b)
 #define BVT(a) __bvt(a,BIHASH_TYPE)
 
+#if _LP64 == 0
+#define OVERFLOW_ASSERT(x) ASSERT(((x) & 0xFFFFFFFF00000000ULL) == 0)
+#define u64_to_pointer(x) (void *)(u32)((x))
+#define pointer_to_u64(x) (u64)(u32)((x))
+#else
+#define OVERFLOW_ASSERT(x)
+#define u64_to_pointer(x) (void *)((x))
+#define pointer_to_u64(x) (u64)((x))
+#endif
+
 typedef struct BV (clib_bihash_value)
 {
   union
   {
     BVT (clib_bihash_kv) kvp[BIHASH_KVP_PER_PAGE];
-    struct BV (clib_bihash_value) * next_free;
+    u64 next_free_as_u64;
   };
 } BVT (clib_bihash_value);
 
@@ -62,7 +83,7 @@ typedef struct
       u64 lock:1;
       u64 linear_search:1;
       u64 log2_pages:8;
-      i64 refcnt:16;
+      u64 refcnt:16;
     };
     u64 as_u64;
   };
@@ -70,9 +91,30 @@ typedef struct
 
 STATIC_ASSERT_SIZEOF (BVT (clib_bihash_bucket), sizeof (u64));
 
+/* *INDENT-OFF* */
+typedef CLIB_PACKED (struct {
+  /*
+   * Backing store allocation. Since bihash manages its own
+   * freelists, we simple dole out memory starting from alloc_arena[alloc_arena_next].
+   */
+  u64 alloc_arena_next;	/* Next offset from alloc_arena to allocate, definitely NOT a constant */
+  u64 alloc_arena_size;	/* Size of the arena */
+  /* Two SVM pointers stored as 8-byte integers */
+  u64 alloc_lock_as_u64;
+  u64 buckets_as_u64;
+  /* freelist list-head arrays/vectors */
+  u64 freelists_as_u64;
+  u32 nbuckets;	/* Number of buckets */
+  /* Set when header valid */
+  volatile u32 ready;
+  u64 pad[2];
+}) BVT (clib_bihash_shared_header);
+/* *INDENT-ON* */
+
+STATIC_ASSERT_SIZEOF (BVT (clib_bihash_shared_header), 8 * sizeof (u64));
+
 typedef struct
 {
-  BVT (clib_bihash_value) * values;
   BVT (clib_bihash_bucket) * buckets;
   volatile u32 *alloc_lock;
 
@@ -84,18 +126,16 @@ typedef struct
   u32 log2_nbuckets;
   u8 *name;
 
-  u64 cache_hits;
-  u64 cache_misses;
+  u64 *freelists;
 
-    BVT (clib_bihash_value) ** freelists;
+#if BIHASH_32_64_SVM
+    BVT (clib_bihash_shared_header) * sh;
+  int memfd;
+#else
+    BVT (clib_bihash_shared_header) sh;
+#endif
 
-  /*
-   * Backing store allocation. Since bihash manages its own
-   * freelists, we simple dole out memory at alloc_arena_next.
-   */
-  uword alloc_arena;
-  uword alloc_arena_next;
-  uword alloc_arena_size;
+  u64 alloc_arena;		/* Base of the allocation arena */
 
   /**
     * A custom format function to print the Key and Value of bihash_key instead of default hexdump
@@ -104,6 +144,26 @@ typedef struct
 
 } BVT (clib_bihash);
 
+#if BIHASH_32_64_SVM
+#undef alloc_arena_next
+#undef alloc_arena_size
+#undef alloc_arena
+#undef CLIB_BIHASH_READY_MAGIC
+#define alloc_arena_next(h) (((h)->sh)->alloc_arena_next)
+#define alloc_arena_size(h) (((h)->sh)->alloc_arena_size)
+#define alloc_arena(h) ((h)->alloc_arena)
+#define CLIB_BIHASH_READY_MAGIC 0xFEEDFACE
+#else
+#undef alloc_arena_next
+#undef alloc_arena_size
+#undef alloc_arena
+#undef CLIB_BIHASH_READY_MAGIC
+#define alloc_arena_next(h) ((h)->sh.alloc_arena_next)
+#define alloc_arena_size(h) ((h)->sh.alloc_arena_size)
+#define alloc_arena(h) ((h)->alloc_arena)
+#define CLIB_BIHASH_READY_MAGIC 0
+#endif
+
 static inline void BV (clib_bihash_alloc_lock) (BVT (clib_bihash) * h)
 {
   while (__atomic_test_and_set (h->alloc_lock, __ATOMIC_ACQUIRE))
@@ -142,7 +202,7 @@ static inline void BV (clib_bihash_unlock_bucket)
 static inline void *BV (clib_bihash_get_value) (BVT (clib_bihash) * h,
 						uword offset)
 {
-  u8 *hp = (u8 *) h->alloc_arena;
+  u8 *hp = (u8 *) (uword) alloc_arena (h);
   u8 *vp = hp + offset;
 
   return (void *) vp;
@@ -160,7 +220,7 @@ static inline uword BV (clib_bihash_get_offset) (BVT (clib_bihash) * h,
 {
   u8 *hp, *vp;
 
-  hp = (u8 *) h->alloc_arena;
+  hp = (u8 *) (uword) alloc_arena (h);
   vp = (u8 *) v;
 
   return vp - hp;
@@ -169,6 +229,13 @@ static inline uword BV (clib_bihash_get_offset) (BVT (clib_bihash) * h,
 void BV (clib_bihash_init)
   (BVT (clib_bihash) * h, char *name, u32 nbuckets, uword memory_size);
 
+#if BIHASH_32_64_SVM
+void BV (clib_bihash_master_init_svm)
+  (BVT (clib_bihash) * h, char *name, u32 nbuckets, u64 memory_size);
+void BV (clib_bihash_slave_init_svm)
+  (BVT (clib_bihash) * h, char *name, int fd);
+#endif
+
 void BV (clib_bihash_set_kvp_format_fn) (BVT (clib_bihash) * h,
 					 format_function_t * fmt_fn);