2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 /** @if DOCUMENTATION_IS_IN_BIHASH_DOC_H */
18 void BV(clib_bihash_init)
19 (BVT(clib_bihash) * h, char * name, u32 nbuckets,
24 nbuckets = 1 << (max_log2 (nbuckets));
27 h->nbuckets = nbuckets;
28 h->log2_nbuckets = max_log2 (nbuckets);
30 h->mheap = mheap_alloc (0 /* use VM */, memory_size);
32 oldheap = clib_mem_set_heap (h->mheap);
33 vec_validate_aligned (h->buckets, nbuckets - 1, CLIB_CACHE_LINE_BYTES);
34 h->writer_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
35 CLIB_CACHE_LINE_BYTES);
37 clib_mem_set_heap (oldheap);
40 void BV(clib_bihash_free) (BVT(clib_bihash) * h)
42 mheap_free (h->mheap);
43 memset (h, 0, sizeof (*h));
46 static BVT(clib_bihash_value) *
47 BV(value_alloc) (BVT(clib_bihash) * h, u32 log2_pages)
49 BVT(clib_bihash_value) * rv = 0;
52 ASSERT (h->writer_lock[0]);
53 if (log2_pages >= vec_len (h->freelists)
54 || h->freelists [log2_pages] == 0)
56 oldheap = clib_mem_set_heap (h->mheap);
58 vec_validate (h->freelists, log2_pages);
59 vec_validate_aligned (rv, (1<<log2_pages) - 1, CLIB_CACHE_LINE_BYTES);
60 clib_mem_set_heap (oldheap);
63 rv = h->freelists[log2_pages];
64 h->freelists[log2_pages] = rv->next_free;
68 ASSERT (vec_len(rv) == (1<<log2_pages));
70 * Latest gcc complains that the length arg is zero
71 * if we replace (1<<log2_pages) with vec_len(rv).
74 memset (rv, 0xff, sizeof (*rv) * (1<<log2_pages));
80 (BVT(clib_bihash) * h,
81 BVT(clib_bihash_value) * v)
85 ASSERT (h->writer_lock[0]);
87 log2_pages = min_log2(vec_len(v));
89 ASSERT(vec_len (h->freelists) > log2_pages);
91 v->next_free = h->freelists[log2_pages];
92 h->freelists[log2_pages] = v;
97 (BVT(clib_bihash) * h, clib_bihash_bucket_t * b)
99 BVT(clib_bihash_value) * v;
100 clib_bihash_bucket_t working_bucket __attribute__((aligned (8)));
102 BVT(clib_bihash_value) * working_copy;
103 u32 cpu_number = os_get_cpu_number();
105 if (cpu_number >= vec_len (h->working_copies))
107 oldheap = clib_mem_set_heap (h->mheap);
108 vec_validate (h->working_copies, cpu_number);
109 clib_mem_set_heap (oldheap);
113 * working_copies are per-cpu so that near-simultaneous
114 * updates from multiple threads will not result in sporadic, spurious
117 working_copy = h->working_copies[cpu_number];
119 h->saved_bucket.as_u64 = b->as_u64;
120 oldheap = clib_mem_set_heap (h->mheap);
122 if ((1<<b->log2_pages) > vec_len (working_copy))
124 vec_validate_aligned (working_copy, (1<<b->log2_pages)-1,
126 h->working_copies[cpu_number] = working_copy;
129 _vec_len(working_copy) = 1<<b->log2_pages;
130 clib_mem_set_heap (oldheap);
132 v = BV(clib_bihash_get_value) (h, b->offset);
134 clib_memcpy (working_copy, v, sizeof (*v)*(1<<b->log2_pages));
135 working_bucket.as_u64 = b->as_u64;
136 working_bucket.offset = BV(clib_bihash_get_offset) (h, working_copy);
137 CLIB_MEMORY_BARRIER();
138 b->as_u64 = working_bucket.as_u64;
139 h->working_copies[cpu_number] = working_copy;
142 static BVT(clib_bihash_value) *
144 (BVT(clib_bihash) * h,
145 BVT(clib_bihash_value) * old_values,
148 BVT(clib_bihash_value) * new_values, * v, * new_v;
151 new_values = BV(value_alloc) (h, new_log2_pages);
154 for (i = 0; i < vec_len (old_values); i++)
158 for (j = 0; j < BIHASH_KVP_PER_PAGE; j++)
160 if (BV(clib_bihash_is_free)(&(v->kvp[j])) == 0)
162 new_hash = BV(clib_bihash_hash) (&(v->kvp[j]));
163 new_hash >>= h->log2_nbuckets;
164 new_hash &= (1<<new_log2_pages) - 1;
166 new_v = &new_values [new_hash];
168 for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
170 if (BV(clib_bihash_is_free)(&(new_v->kvp[k])))
172 clib_memcpy (&(new_v->kvp[k]), &(v->kvp[j]),
173 sizeof (new_v->kvp[k]));
177 /* Crap. Tell caller to try again */
178 BV(value_free) (h, new_values);
189 int BV(clib_bihash_add_del)
190 (BVT(clib_bihash) * h,
191 BVT(clib_bihash_kv) * add_v,
195 clib_bihash_bucket_t * b, tmp_b;
196 BVT(clib_bihash_value) * v, * new_v, * save_new_v, * working_copy;
202 u32 cpu_number = os_get_cpu_number();
204 hash = BV(clib_bihash_hash) (add_v);
206 bucket_index = hash & (h->nbuckets-1);
207 b = &h->buckets[bucket_index];
209 hash >>= h->log2_nbuckets;
211 while (__sync_lock_test_and_set (h->writer_lock, 1))
214 /* First elt in the bucket? */
223 v = BV(value_alloc) (h, 0);
226 tmp_b.offset = BV(clib_bihash_get_offset) (h, v);
228 b->as_u64 = tmp_b.as_u64;
232 BV(make_working_copy) (h, b);
234 v = BV(clib_bihash_get_value) (h, h->saved_bucket.offset);
235 value_index = hash & ((1<<h->saved_bucket.log2_pages)-1);
241 * For obvious (in hindsight) reasons, see if we're supposed to
242 * replace an existing key, then look for an empty slot.
244 for (i = 0; i < BIHASH_KVP_PER_PAGE; i++)
246 if (!memcmp(&(v->kvp[i]), &add_v->key, sizeof (add_v->key)))
248 clib_memcpy (&(v->kvp[i]), add_v, sizeof (*add_v));
249 CLIB_MEMORY_BARRIER();
250 /* Restore the previous (k,v) pairs */
251 b->as_u64 = h->saved_bucket.as_u64;
255 for (i = 0; i < BIHASH_KVP_PER_PAGE; i++)
257 if (BV(clib_bihash_is_free)(&(v->kvp[i])))
259 clib_memcpy (&(v->kvp[i]), add_v, sizeof (*add_v));
260 CLIB_MEMORY_BARRIER();
261 b->as_u64 = h->saved_bucket.as_u64;
265 /* no room at the inn... split case... */
269 for (i = 0; i < BIHASH_KVP_PER_PAGE; i++)
271 if (!memcmp(&(v->kvp[i]), &add_v->key, sizeof (add_v->key)))
273 memset (&(v->kvp[i]), 0xff, sizeof (*(add_v)));
274 CLIB_MEMORY_BARRIER();
275 b->as_u64 = h->saved_bucket.as_u64;
280 b->as_u64 = h->saved_bucket.as_u64;
284 new_log2_pages = h->saved_bucket.log2_pages + 1;
287 working_copy = h->working_copies[cpu_number];
288 new_v = BV(split_and_rehash) (h, working_copy, new_log2_pages);
295 /* Try to add the new entry */
297 new_hash = BV(clib_bihash_hash) (add_v);
298 new_hash >>= h->log2_nbuckets;
299 new_hash &= (1<<min_log2(vec_len(new_v))) - 1;
302 for (i = 0; i < BIHASH_KVP_PER_PAGE; i++)
304 if (BV(clib_bihash_is_free)(&(new_v->kvp[i])))
306 clib_memcpy (&(new_v->kvp[i]), add_v, sizeof (*add_v));
310 /* Crap. Try again */
312 BV(value_free) (h, save_new_v);
316 tmp_b.log2_pages = min_log2 (vec_len (save_new_v));
317 tmp_b.offset = BV(clib_bihash_get_offset) (h, save_new_v);
318 CLIB_MEMORY_BARRIER();
319 b->as_u64 = tmp_b.as_u64;
320 v = BV(clib_bihash_get_value) (h, h->saved_bucket.offset);
321 BV(value_free) (h, v);
324 CLIB_MEMORY_BARRIER();
325 h->writer_lock[0] = 0;
329 int BV(clib_bihash_search)
330 (BVT(clib_bihash) * h,
331 BVT(clib_bihash_kv) *search_key,
332 BVT(clib_bihash_kv) *valuep)
337 BVT(clib_bihash_value) * v;
338 clib_bihash_bucket_t * b;
343 hash = BV(clib_bihash_hash) (search_key);
345 bucket_index = hash & (h->nbuckets-1);
346 b = &h->buckets[bucket_index];
351 hash >>= h->log2_nbuckets;
353 v = BV(clib_bihash_get_value) (h, b->offset);
354 value_index = hash & ((1<<b->log2_pages)-1);
357 for (i = 0; i < BIHASH_KVP_PER_PAGE; i++)
359 if (BV(clib_bihash_key_compare)(v->kvp[i].key, search_key->key))
368 u8 * BV(format_bihash) (u8 * s, va_list * args)
371 = va_arg (*args, BVT(clib_bihash) *);
372 int verbose = va_arg (*args, int);
373 clib_bihash_bucket_t * b;
374 BVT(clib_bihash_value) * v;
376 u64 active_elements = 0;
378 s = format (s, "Hash table %s\n", h->name ? h->name : (u8 *) "(unnamed)");
380 for (i = 0; i < h->nbuckets; i++)
386 s = format (s, "[%d]: empty\n", i);
392 s = format (s, "[%d]: heap offset %d, len %d\n", i,
393 b->offset, (1<<b->log2_pages));
396 v = BV(clib_bihash_get_value) (h, b->offset);
397 for (j = 0; j < (1<<b->log2_pages); j++)
399 for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
401 if (BV(clib_bihash_is_free)(&v->kvp[k]))
404 s = format (s, " %d: empty\n",
405 j * BIHASH_KVP_PER_PAGE + k);
410 s = format (s, " %d: %U\n",
411 j * BIHASH_KVP_PER_PAGE + k,
412 BV(format_bihash_kvp),
421 s = format (s, " %lld active elements\n", active_elements);
422 s = format (s, " %d free lists\n", vec_len (h->freelists));
427 void BV(clib_bihash_foreach_key_value_pair)
428 (BVT(clib_bihash) * h,
433 clib_bihash_bucket_t * b;
434 BVT(clib_bihash_value) * v;
435 void (*fp)(BVT(clib_bihash_kv) *, void *) = callback;
437 for (i = 0; i < h->nbuckets; i++)
443 v = BV(clib_bihash_get_value) (h, b->offset);
444 for (j = 0; j < (1<<b->log2_pages); j++)
446 for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
448 if (BV(clib_bihash_is_free)(&v->kvp[k]))
451 (*fp)(&v->kvp[k], arg);