static inline void BV (clib_bihash_lock_bucket) (BVT (clib_bihash_bucket) * b)
{
- BVT (clib_bihash_bucket) unlocked_bucket, locked_bucket;
+ /* *INDENT-OFF* */
+ BVT (clib_bihash_bucket) mask = { .lock = 1 };
+ /* *INDENT-ON* */
+ u64 old;
- do
+try_again:
+ old = clib_atomic_fetch_or (&b->as_u64, mask.as_u64);
+
+ if (PREDICT_FALSE (old & mask.as_u64))
{
- locked_bucket.as_u64 = unlocked_bucket.as_u64 = b->as_u64;
- unlocked_bucket.lock = 0;
- locked_bucket.lock = 1;
+ /* somebody else flipped the bit, try again */
CLIB_PAUSE ();
+ goto try_again;
}
- while (__atomic_compare_exchange_n (&b->as_u64, &unlocked_bucket.as_u64,
- locked_bucket.as_u64, 1 /* weak */ ,
- __ATOMIC_ACQUIRE,
- __ATOMIC_ACQUIRE) == 0);
}
static inline void BV (clib_bihash_unlock_bucket)
(BVT (clib_bihash_bucket) * b)
{
- CLIB_MEMORY_BARRIER ();
b->lock = 0;
}
void BV (clib_bihash_init2) (BVT (clib_bihash_init2_args) * a);
#if BIHASH_32_64_SVM
-void BV (clib_bihash_master_init_svm)
+void BV (clib_bihash_initiator_init_svm)
(BVT (clib_bihash) * h, char *name, u32 nbuckets, u64 memory_size);
-void BV (clib_bihash_slave_init_svm)
+void BV (clib_bihash_responder_init_svm)
(BVT (clib_bihash) * h, char *name, int fd);
#endif
offset = offset * (sizeof (BVT (clib_bihash_bucket))
+ (BIHASH_KVP_PER_PAGE * sizeof (BVT (clib_bihash_kv))));
return ((BVT (clib_bihash_bucket) *) (((u8 *) h->buckets) + offset));
-#endif
-
+#else
return h->buckets + (hash & (h->nbuckets - 1));
+#endif
}
static inline int BV (clib_bihash_search_inline_with_hash)
BVT (clib_bihash_bucket) * b;
int i, limit;
+ /* *INDENT-OFF* */
+ static const BVT (clib_bihash_bucket) mask = {
+ .linear_search = 1,
+ .log2_pages = -1
+ };
+ /* *INDENT-ON* */
+
#if BIHASH_LAZY_INSTANTIATE
- if (PREDICT_FALSE (alloc_arena (h) == 0))
+ if (PREDICT_FALSE (h->instantiated == 0))
return -1;
#endif
CLIB_PAUSE ();
}
- hash >>= h->log2_nbuckets;
-
v = BV (clib_bihash_get_value) (h, b->offset);
/* If the bucket has unresolvable collisions, use linear search */
limit = BIHASH_KVP_PER_PAGE;
- v += (b->linear_search == 0) ? hash & ((1 << b->log2_pages) - 1) : 0;
- if (PREDICT_FALSE (b->linear_search))
- limit <<= b->log2_pages;
+
+ if (PREDICT_FALSE (b->as_u64 & mask.as_u64))
+ {
+ if (PREDICT_FALSE (b->linear_search))
+ limit <<= b->log2_pages;
+ else
+ v += extract_bits (hash, h->log2_nbuckets, b->log2_pages);
+ }
for (i = 0; i < limit; i++)
{
BVT (clib_bihash_bucket) * b;
#if BIHASH_LAZY_INSTANTIATE
- if (PREDICT_FALSE (alloc_arena (h) == 0))
+ if (PREDICT_FALSE (h->instantiated == 0))
return;
#endif
if (PREDICT_FALSE (BV (clib_bihash_bucket_is_empty) (b)))
return;
- hash >>= h->log2_nbuckets;
v = BV (clib_bihash_get_value) (h, b->offset);
- v += (b->linear_search == 0) ? hash & ((1 << b->log2_pages) - 1) : 0;
+ if (PREDICT_FALSE (b->log2_pages && b->linear_search == 0))
+ v += extract_bits (hash, h->log2_nbuckets, b->log2_pages);
- clib_prefetch_load (v);
+ CLIB_PREFETCH (v, BIHASH_KVP_PER_PAGE * sizeof (BVT (clib_bihash_kv)),
+ LOAD);
}
static inline int BV (clib_bihash_search_inline_2_with_hash)
BVT (clib_bihash_bucket) * b;
int i, limit;
+/* *INDENT-OFF* */
+ static const BVT (clib_bihash_bucket) mask = {
+ .linear_search = 1,
+ .log2_pages = -1
+ };
+/* *INDENT-ON* */
+
ASSERT (valuep);
#if BIHASH_LAZY_INSTANTIATE
- if (PREDICT_FALSE (alloc_arena (h) == 0))
+ if (PREDICT_FALSE (h->instantiated == 0))
return -1;
#endif
CLIB_PAUSE ();
}
- hash >>= h->log2_nbuckets;
v = BV (clib_bihash_get_value) (h, b->offset);
/* If the bucket has unresolvable collisions, use linear search */
limit = BIHASH_KVP_PER_PAGE;
- v += (b->linear_search == 0) ? hash & ((1 << b->log2_pages) - 1) : 0;
- if (PREDICT_FALSE (b->linear_search))
- limit <<= b->log2_pages;
+
+ if (PREDICT_FALSE (b->as_u64 & mask.as_u64))
+ {
+ if (PREDICT_FALSE (b->linear_search))
+ limit <<= b->log2_pages;
+ else
+ v += extract_bits (hash, h->log2_nbuckets, b->log2_pages);
+ }
for (i = 0; i < limit; i++)
{