X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fbihash_template.c;h=2b378427ce8c79e325d64ae700a0fff4e8e153f6;hb=a690fdbfe179e0ea65818c03b52535bf9210efd0;hp=cd75a7a9be97a7812453c83fc8db98b0b81fed98;hpb=9e4946b324a8480b35c861e96199881b314ea825;p=vpp.git diff --git a/src/vppinfra/bihash_template.c b/src/vppinfra/bihash_template.c index cd75a7a9be9..2b378427ce8 100644 --- a/src/vppinfra/bihash_template.c +++ b/src/vppinfra/bihash_template.c @@ -26,13 +26,13 @@ static inline void *BV (alloc_aligned) (BVT (clib_bihash) * h, uword nbytes) rv = alloc_arena_next (h); alloc_arena_next (h) += nbytes; - if (rv >= alloc_arena_size (h)) + if (alloc_arena_next (h) > alloc_arena_size (h)) os_out_of_memory (); return (void *) (uword) (rv + alloc_arena (h)); } -void BV (clib_bihash_instantiate) (BVT (clib_bihash) * h) +static void BV (clib_bihash_instantiate) (BVT (clib_bihash) * h) { uword bucket_size; @@ -42,22 +42,25 @@ void BV (clib_bihash_instantiate) (BVT (clib_bihash) * h) bucket_size = h->nbuckets * sizeof (h->buckets[0]); h->buckets = BV (alloc_aligned) (h, bucket_size); - - h->alloc_lock = BV (alloc_aligned) (h, CLIB_CACHE_LINE_BYTES); - h->alloc_lock[0] = 0; + CLIB_MEMORY_BARRIER (); + h->instantiated = 1; } -void BV (clib_bihash_init) - (BVT (clib_bihash) * h, char *name, u32 nbuckets, uword memory_size) +void BV (clib_bihash_init2) (BVT (clib_bihash_init2_args) * a) { int i; void *oldheap; - nbuckets = 1 << (max_log2 (nbuckets)); + BVT (clib_bihash) * h = a->h; + + a->nbuckets = 1 << (max_log2 (a->nbuckets)); + + h->name = (u8 *) a->name; + h->nbuckets = a->nbuckets; + h->log2_nbuckets = max_log2 (a->nbuckets); + h->memory_size = a->memory_size; + h->instantiated = 0; + h->fmt_fn = a->fmt_fn; - h->name = (u8 *) name; - h->nbuckets = nbuckets; - h->log2_nbuckets = max_log2 (nbuckets); - h->memory_size = memory_size; alloc_arena (h) = 0; /* @@ -66,22 +69,48 @@ void BV (clib_bihash_init) * If someone starts complaining that's not enough, we can shift * the offset by CLIB_LOG2_CACHE_LINE_BYTES... */ - ASSERT (memory_size < (1ULL << BIHASH_BUCKET_OFFSET_BITS)); - h->fmt_fn = NULL; + ASSERT (h->memory_size < (1ULL << BIHASH_BUCKET_OFFSET_BITS)); /* Add this hash table to the list */ - for (i = 0; i < vec_len (clib_all_bihashes); i++) - if (clib_all_bihashes[i] == h) - return; + if (a->dont_add_to_all_bihash_list == 0) + { + for (i = 0; i < vec_len (clib_all_bihashes); i++) + if (clib_all_bihashes[i] == h) + goto do_lock; + oldheap = clib_all_bihash_set_heap (); + vec_add1 (clib_all_bihashes, (void *) h); + clib_mem_set_heap (oldheap); + } - /* Unfortunately, the heap push/pop is required.... */ - oldheap = clib_all_bihash_set_heap (); - vec_add1 (clib_all_bihashes, (void *) h); - clib_mem_set_heap (oldheap); +do_lock: + if (h->alloc_lock) + clib_mem_free ((void *) h->alloc_lock); -#if BIHASH_INSTANTIATE_IMMEDIATELY - BV (clib_bihash_instantiate) (h); -#endif + /* + * Set up the lock now, so we can use it to make the first add + * thread-safe + */ + h->alloc_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, + CLIB_CACHE_LINE_BYTES); + h->alloc_lock[0] = 0; + + if (a->instantiate_immediately) + BV (clib_bihash_instantiate) (h); +} + +void BV (clib_bihash_init) + (BVT (clib_bihash) * h, char *name, u32 nbuckets, uword memory_size) +{ + BVT (clib_bihash_init2_args) _a, *a = &_a; + + memset (a, 0, sizeof (*a)); + + a->h = h; + a->name = name; + a->nbuckets = nbuckets; + a->memory_size = memory_size; + + BV (clib_bihash_init2) (a); } #if BIHASH_32_64_SVM @@ -150,12 +179,12 @@ void BV (clib_bihash_master_init_svm) sizeof (vec_header_t) + BIHASH_FREELIST_LENGTH * sizeof (u64)); freelist_vh->len = BIHASH_FREELIST_LENGTH; - freelist_vh->dlmalloc_header_offset = 0xDEADBEEF; h->sh->freelists_as_u64 = (u64) BV (clib_bihash_get_offset) (h, freelist_vh->vector_data); h->freelists = (void *) (freelist_vh->vector_data); h->fmt_fn = NULL; + h->instantiated = 1; } void BV (clib_bihash_slave_init_svm) @@ -216,9 +245,10 @@ void BV (clib_bihash_free) (BVT (clib_bihash) * h) { int i; - if (PREDICT_FALSE (alloc_arena (h) == 0)) + if (PREDICT_FALSE (h->instantiated == 0)) goto never_initialized; + h->instantiated = 0; vec_free (h->working_copies); vec_free (h->working_copy_lengths); #if BIHASH_32_64_SVM == 0 @@ -239,7 +269,7 @@ never_initialized: } } clib_warning ("Couldn't find hash table %llx on clib_all_bihashes...", - (u64) h); + (u64) (uword) h); } static @@ -452,12 +482,19 @@ static inline int BV (clib_bihash_add_del_inline) int mark_bucket_linear; int resplit_once; - /* Create the table (is_add=1), or flunk the request now (is_add=0) */ - if (PREDICT_FALSE (alloc_arena (h) == 0)) + /* + * Create the table (is_add=1,2), or flunk the request now (is_add=0) + * Use the alloc_lock to protect the instantiate operation. + */ + if (PREDICT_FALSE (h->instantiated == 0)) { if (is_add == 0) return (-1); - BV (clib_bihash_instantiate) (h); + + BV (clib_bihash_alloc_lock) (h); + if (h->instantiated == 0) + BV (clib_bihash_instantiate) (h); + BV (clib_bihash_alloc_unlock) (h); } hash = BV (clib_bihash_hash) (add_v); @@ -869,12 +906,12 @@ u8 *BV (format_bihash) (u8 * s, va_list * args) } void BV (clib_bihash_foreach_key_value_pair) - (BVT (clib_bihash) * h, void *callback, void *arg) + (BVT (clib_bihash) * h, + BV (clib_bihash_foreach_key_value_pair_cb) cb, void *arg) { int i, j, k; BVT (clib_bihash_bucket) * b; BVT (clib_bihash_value) * v; - void (*fp) (BVT (clib_bihash_kv) *, void *) = callback; if (PREDICT_FALSE (alloc_arena (h) == 0)) return; @@ -893,7 +930,8 @@ void BV (clib_bihash_foreach_key_value_pair) if (BV (clib_bihash_is_free) (&v->kvp[k])) continue; - (*fp) (&v->kvp[k], arg); + if (BIHASH_WALK_STOP == cb (&v->kvp[k], arg)) + return; /* * In case the callback deletes the last entry in the bucket... */