2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vlib/vlib.h>
16 #include <vppinfra/time.h>
17 #include <vppinfra/cache.h>
18 #include <vppinfra/error.h>
19 #include <sys/resource.h>
23 #include <vppinfra/bihash_8_8_stats.h>
24 #include <vppinfra/bihash_template.h>
26 #include <vppinfra/bihash_template.c>
39 u64 working_copy_lost;
45 volatile u32 thread_barrier;
46 volatile u32 threads_running;
47 volatile u64 sequence_number;
54 int careful_delete_tests;
60 uword hash_memory_size;
61 BVT (clib_bihash) hash;
62 clib_time_t clib_time;
65 unformat_input_t *input;
68 vlib_main_t *vlib_main;
70 CLIB_CACHE_LINE_ALIGN_MARK (stat_align);
75 static bihash_test_main_t bihash_test_main;
78 format_bihash_stats (u8 * s, va_list * args)
80 BVT (clib_bihash) * h = va_arg (*args, BVT (clib_bihash) *);
81 int verbose = va_arg (*args, int);
83 bihash_stats_t *sp = h->inc_stats_context;
85 #define _(a) s = format (s, "%20s: %lld\n", #a, sp->a);
88 for (i = 0; i < vec_len (sp->splits); i++)
90 if (sp->splits[i] > 0 || verbose)
91 s = format (s, " splits[%d]: %lld\n", 1 << i, sp->splits[i]);
98 inc_stats_callback (BVT (clib_bihash) * h, int stat_id, u64 count)
100 uword *statp = h->inc_stats_context;
101 bihash_stats_t *for_splits;
103 if (PREDICT_TRUE (stat_id * sizeof (u64)
104 < STRUCT_OFFSET_OF (bihash_stats_t, splits)))
106 statp[stat_id] += count;
110 for_splits = h->inc_stats_context;
111 vec_validate (for_splits->splits, count);
112 for_splits->splits[count] += 1;
116 static clib_error_t *
117 test_bihash_vec64 (bihash_test_main_t * tm)
119 u32 user_buckets = 1228800;
120 u32 user_memory_size = 209715200;
121 BVT (clib_bihash_kv) kv;
125 BVT (clib_bihash) * h;
129 BV (clib_bihash_init) (h, "test", user_buckets, user_memory_size);
130 BV (clib_bihash_set_stats_callback) (h, inc_stats_callback, &tm->stats);
132 before = clib_time_now (&tm->clib_time);
134 for (j = 0; j < 10; j++)
136 for (i = 1; i <= j * 1000 + 1; i++)
141 BV (clib_bihash_add_del) (h, &kv, 1 /* is_add */ );
144 vec_add1 (cum_times, clib_time_now (&tm->clib_time) - before);
147 for (j = 0; j < vec_len (cum_times); j++)
148 fformat (stdout, "Cum time for %d: %.4f (us)\n", (j + 1) * 1000,
155 test_bihash_thread_fn (void *arg)
157 BVT (clib_bihash) * h;
158 BVT (clib_bihash_kv) kv;
159 bihash_test_main_t *tm = &bihash_test_main;
161 u32 my_thread_index = (uword) arg;
163 while (tm->thread_barrier)
168 for (i = 0; i < tm->ncycles; i++)
170 for (j = 0; j < tm->nitems; j++)
172 kv.key = ((u64) my_thread_index << 32) | (u64) j;
173 kv.value = ((u64) my_thread_index << 32) | (u64) j;
174 (void) __atomic_add_fetch (&tm->sequence_number, 1,
176 BV (clib_bihash_add_del) (h, &kv, 1 /* is_add */ );
178 for (j = 0; j < tm->nitems; j++)
180 kv.key = ((u64) my_thread_index << 32) | (u64) j;
181 kv.value = ((u64) my_thread_index << 32) | (u64) j;
182 (void) __atomic_add_fetch (&tm->sequence_number, 1,
184 BV (clib_bihash_add_del) (h, &kv, 0 /* is_add */ );
188 (void) __atomic_sub_fetch (&tm->threads_running, 1, __ATOMIC_ACQUIRE);
190 return (0); /* not so much */
193 static clib_error_t *
194 test_bihash_threads (bihash_test_main_t * tm)
198 BVT (clib_bihash) * h;
200 f64 before, after, delta;
204 BV (clib_bihash_init) (h, "test", tm->nbuckets, tm->hash_memory_size);
205 BV (clib_bihash_set_stats_callback) (h, inc_stats_callback, &tm->stats);
207 tm->thread_barrier = 1;
209 /* Start the worker threads */
210 for (i = 0; i < tm->nthreads; i++)
212 rv = pthread_create (&handle, NULL, test_bihash_thread_fn,
216 clib_unix_warning ("pthread_create returned %d", rv);
219 tm->threads_running = i;
220 tm->sequence_number = 0;
221 CLIB_MEMORY_BARRIER ();
223 /* start the workers */
224 before = vlib_time_now (tm->vlib_main);
225 tm->thread_barrier = 0;
227 while (tm->threads_running > 0)
230 after = vlib_time_now (tm->vlib_main);
231 delta = after - before;
233 clib_warning ("%lld items in %.2f seconds, %.2f items/sec",
234 (u64) tm->nthreads * (u64) tm->nitems, delta,
236 0.0 ? ((f64) ((u64) tm->nthreads * (u64) tm->nitems)) /
239 fformat (stdout, "Stats:\n%U", format_bihash_stats, h, 1 /* verbose */ );
240 BV (clib_bihash_free) (h);
245 * Callback to blow up spectacularly if anything remains in the table
248 count_items (BVT (clib_bihash_kv) * kvp, void *notused)
250 _clib_error (CLIB_ERROR_ABORT, 0, 0,
251 "bihash test FAILED, items left in table!");
254 static clib_error_t *
255 test_bihash (bihash_test_main_t * tm)
259 uword total_searches;
261 BVT (clib_bihash) * h;
262 BVT (clib_bihash_kv) kv;
267 BV (clib_bihash_init) (h, "test", tm->nbuckets, tm->hash_memory_size);
268 BV (clib_bihash_set_stats_callback) (h, inc_stats_callback, &tm->stats);
270 for (acycle = 0; acycle < tm->ncycles; acycle++)
272 if ((acycle % tm->report_every_n) == 0)
274 fformat (stdout, "Cycle %lld out of %lld...\n",
275 acycle, tm->ncycles);
277 fformat (stdout, "Pick %lld unique %s keys...\n",
278 tm->nitems, tm->non_random_keys ? "non-random" : "random");
281 for (i = 0; i < tm->nitems; i++)
285 if (tm->non_random_keys == 0)
289 rndkey = random_u64 (&tm->seed);
291 p = hash_get (tm->key_hash, rndkey);
296 rndkey = (u64) (i + 1) << 16;
299 hash_set (tm->key_hash, rndkey, i + 1);
300 vec_add1 (tm->keys, rndkey);
304 if ((acycle % tm->report_every_n) == 0)
305 fformat (stdout, "Add items...\n");
307 for (i = 0; i < tm->nitems; i++)
309 kv.key = tm->keys[i];
312 BV (clib_bihash_add_del) (h, &kv, 1 /* is_add */ );
316 fformat (stdout, "--------------------\n");
317 fformat (stdout, "After adding key %llu value %lld...\n",
318 tm->keys[i], (u64) (i + 1));
319 fformat (stdout, "%U", BV (format_bihash), h,
320 2 /* very verbose */ );
324 if ((acycle % tm->report_every_n) == 0)
326 fformat (stdout, "%U", BV (format_bihash), h,
327 0 /* very verbose */ );
329 fformat (stdout, "Search for items %d times...\n", tm->search_iter);
332 before = clib_time_now (&tm->clib_time);
334 for (j = 0; j < tm->search_iter; j++)
336 for (i = 0; i < tm->nitems; i++)
338 kv.key = tm->keys[i];
339 if (BV (clib_bihash_search) (h, &kv, &kv) < 0)
340 if (BV (clib_bihash_search) (h, &kv, &kv) < 0)
342 ("[%d] search for key %lld failed unexpectedly\n", i,
344 if (kv.value != (u64) (i + 1))
346 ("[%d] search for key %lld returned %lld, not %lld\n", i,
347 tm->keys, kv.value, (u64) (i + 1));
351 if ((acycle % tm->report_every_n) == 0)
353 delta = clib_time_now (&tm->clib_time) - before;
354 total_searches = (uword) tm->search_iter * (uword) tm->nitems;
357 fformat (stdout, "%.f searches per second\n",
358 ((f64) total_searches) / delta);
360 fformat (stdout, "%lld searches in %.6f seconds\n", total_searches,
363 fformat (stdout, "Standard E-hash search for items %d times...\n",
367 before = clib_time_now (&tm->clib_time);
369 for (j = 0; j < tm->search_iter; j++)
371 for (i = 0; i < tm->nitems; i++)
373 p = hash_get (tm->key_hash, tm->keys[i]);
374 if (p == 0 || p[0] != (uword) (i + 1))
375 clib_warning ("ugh, couldn't find %lld\n", tm->keys[i]);
379 delta = clib_time_now (&tm->clib_time) - before;
380 total_searches = (uword) tm->search_iter * (uword) tm->nitems;
382 if ((acycle % tm->report_every_n) == 0)
384 fformat (stdout, "%lld searches in %.6f seconds\n",
385 total_searches, delta);
388 fformat (stdout, "%.f searches per second\n",
389 ((f64) total_searches) / delta);
390 fformat (stdout, "Delete items...\n");
393 for (i = 0; i < tm->nitems; i++)
398 kv.key = tm->keys[i];
399 kv.value = (u64) (i + 1);
400 rv = BV (clib_bihash_add_del) (h, &kv, 0 /* is_add */ );
403 clib_warning ("delete key %lld not ok but should be",
406 if (tm->careful_delete_tests)
408 for (j = 0; j < tm->nitems; j++)
410 kv.key = tm->keys[j];
411 rv = BV (clib_bihash_search) (h, &kv, &kv);
412 if (j <= i && rv >= 0)
415 ("i %d j %d search ok but should not be, value %lld",
420 clib_warning ("i %d j %d search not ok but should be",
426 if ((acycle % tm->report_every_n) == 0)
428 struct rusage r_usage;
429 getrusage (RUSAGE_SELF, &r_usage);
430 fformat (stdout, "Kernel RSS: %ld bytes\n", r_usage.ru_maxrss);
431 fformat (stdout, "%U\n", BV (format_bihash), h, 0 /* verbose */ );
434 /* Clean up side-bet hash table and random key vector */
435 hash_free (tm->key_hash);
436 vec_reset_length (tm->keys);
437 /* Recreate hash table if we're going to need it again */
438 if (acycle != (tm->ncycles - 1))
439 tm->key_hash = hash_create (tm->nitems, sizeof (uword));
442 fformat (stdout, "End of run, should be empty...\n");
444 fformat (stdout, "%U", BV (format_bihash), h, 0 /* very verbose */ );
446 /* ASSERTs if any items remain */
447 BV (clib_bihash_foreach_key_value_pair) (h, count_items, 0);
449 fformat (stdout, "Stats:\n%U", format_bihash_stats, h, 1 /* verbose */ );
451 BV (clib_bihash_free) (h);
454 hash_free (tm->key_hash);
459 static clib_error_t *
460 test_bihash_command_fn (vlib_main_t * vm,
461 unformat_input_t * input, vlib_cli_command_t * cmd)
463 bihash_test_main_t *tm = &bihash_test_main;
467 tm->hash_memory_size = 32ULL << 20;
468 tm->nbuckets = 32000;
471 tm->report_every_n = 50000;
472 tm->seed = 0x1badf00d;
474 memset (&tm->stats, 0, sizeof (tm->stats));
476 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
478 if (unformat (input, "seed %u", &tm->seed))
481 else if (unformat (input, "nbuckets %d", &tm->nbuckets))
483 else if (unformat (input, "non-random-keys"))
484 tm->non_random_keys = 1;
485 else if (unformat (input, "nitems %d", &tm->nitems))
487 else if (unformat (input, "ncycles %d", &tm->ncycles))
489 else if (unformat (input, "careful %d", &tm->careful_delete_tests))
491 else if (unformat (input, "verbose %d", &tm->verbose))
493 else if (unformat (input, "search %d", &tm->search_iter))
495 else if (unformat (input, "report-every %d", &tm->report_every_n))
497 else if (unformat (input, "memory-size %U",
498 unformat_memory_size, &tm->hash_memory_size))
500 else if (unformat (input, "vec64"))
502 else if (unformat (input, "threads %u", &tm->nthreads))
504 else if (unformat (input, "verbose"))
507 return clib_error_return (0, "unknown input '%U'",
508 format_unformat_error, input);
511 /* Preallocate hash table, key vector */
512 tm->key_hash = hash_create (tm->nitems, sizeof (uword));
513 vec_validate (tm->keys, tm->nitems - 1);
514 _vec_len (tm->keys) = 0;
519 error = test_bihash (tm);
523 error = test_bihash_vec64 (tm);
527 error = test_bihash_threads (tm);
531 return clib_error_return (0, "no such test?");
538 VLIB_CLI_COMMAND (test_bihash_command, static) =
540 .path = "test bihash",
541 .short_help = "test bihash",
542 .function = test_bihash_command_fn,
546 static clib_error_t *
547 bihash_test_init (vlib_main_t * vm)
549 bihash_test_main_t *tm = &bihash_test_main;
555 VLIB_INIT_FUNCTION (bihash_test_init);
558 * fd.io coding-style-patch-verification: ON
561 * eval: (c-set-style "gnu")