src/vppinfra/test_vhash.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16   Copyright (c) 2010 Eliot Dresselhaus
  17
  18   Permission is hereby granted, free of charge, to any person obtaining
  19   a copy of this software and associated documentation files (the
  20   "Software"), to deal in the Software without restriction, including
  21   without limitation the rights to use, copy, modify, merge, publish,
  22   distribute, sublicense, and/or sell copies of the Software, and to
  23   permit persons to whom the Software is furnished to do so, subject to
  24   the following conditions:
  25
  26   The above copyright notice and this permission notice shall be
  27   included in all copies or substantial portions of the Software.
  28
  29   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  30   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  31   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  32   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  33   LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  34   OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  35   WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  36 */
  37
  38 #if 0
  39 #ifdef __OPTIMIZE__
  40 #undef CLIB_DEBUG
  41 #endif
  42 #endif
  43
  44 #include <vppinfra/bitmap.h>
  45 #include <vppinfra/error.h>
  46 #include <vppinfra/os.h>
  47 #include <vppinfra/random.h>
  48 #include <vppinfra/time.h>
  49 #include <vppinfra/vhash.h>
  50
  51 #ifdef CLIB_HAVE_VEC128
  52
  53 typedef struct
  54 {
  55   u32 n_iter;
  56   u32 seed;
  57   u32 verbose;
  58   u32 n_keys;
  59   u32 log2_size;
  60   u32 n_key_u32;
  61
  62   u32 n_vectors_div_4;
  63   u32 n_vectors_mod_4;
  64
  65   u32 *keys;
  66   u32 *results;
  67
  68   u32 *vhash_get_key_indices;
  69   u32 *vhash_get_results;
  70
  71   u32 *vhash_key_indices;
  72   u32 *vhash_results;
  73
  74   vhash_t vhash;
  75
  76   uword **key_hash;
  77
  78   struct
  79   {
  80     u64 n_clocks;
  81     u64 n_vectors;
  82     u64 n_calls;
  83   } get_stats, set_stats, unset_stats;
  84 } test_vhash_main_t;
  85
  86 always_inline u32
  87 test_vhash_key_gather (void *_tm, u32 vi, u32 wi, u32 n_key_u32s)
  88 {
  89   test_vhash_main_t *tm = _tm;
  90   ASSERT (n_key_u32s == tm->n_key_u32);
  91   ASSERT (wi < n_key_u32s);
  92   vi = vec_elt (tm->vhash_key_indices, vi);
  93   return vec_elt (tm->keys, vi * n_key_u32s + wi);
  94 }
  95
  96 always_inline u32x4
  97 test_vhash_4key_gather (void *_tm, u32 vi, u32 wi, u32 n_key_u32s)
  98 {
  99   test_vhash_main_t *tm = _tm;
 100   u32 *p;
 101   u32x4_union_t x;
 102
 103   ASSERT (n_key_u32s == tm->n_key_u32);
 104   ASSERT (wi < n_key_u32s);
 105
 106   p = vec_elt_at_index (tm->vhash_key_indices, vi + 0);
 107   x.as_u32[0] = tm->keys[p[0] * n_key_u32s + wi];
 108   x.as_u32[1] = tm->keys[p[1] * n_key_u32s + wi];
 109   x.as_u32[2] = tm->keys[p[2] * n_key_u32s + wi];
 110   x.as_u32[3] = tm->keys[p[3] * n_key_u32s + wi];
 111   return x.as_u32x4;
 112 }
 113
 114 always_inline u32
 115 test_vhash_get_result (void *_tm,
 116                        u32 vector_index, u32 result_index, u32 n_key_u32s)
 117 {
 118   test_vhash_main_t *tm = _tm;
 119   u32 *p = vec_elt_at_index (tm->vhash_results, vector_index);
 120   p[0] = result_index;
 121   return result_index;
 122 }
 123
 124 always_inline u32x4
 125 test_vhash_get_4result (void *_tm,
 126                         u32 vector_index, u32x4 results, u32 n_key_u32s)
 127 {
 128   test_vhash_main_t *tm = _tm;
 129   u32 *p = vec_elt_at_index (tm->vhash_results, vector_index);
 130   *(u32x4 *) p = results;
 131   return results;
 132 }
 133
 134 always_inline u32
 135 test_vhash_set_result (void *_tm,
 136                        u32 vector_index, u32 old_result, u32 n_key_u32s)
 137 {
 138   test_vhash_main_t *tm = _tm;
 139   u32 *p = vec_elt_at_index (tm->vhash_results, vector_index);
 140   u32 new_result = p[0];
 141   p[0] = old_result;
 142   return new_result;
 143 }
 144
 145 always_inline u32
 146 test_vhash_unset_result (void *_tm, u32 i, u32 old_result, u32 n_key_u32s)
 147 {
 148   test_vhash_main_t *tm = _tm;
 149   u32 *p = vec_elt_at_index (tm->vhash_results, i);
 150   p[0] = old_result;
 151   return 0;
 152 }
 153
 154 #define _(N_KEY_U32)                                                    \
 155   always_inline u32                                                     \
 156   test_vhash_key_gather_##N_KEY_U32 (void * _tm, u32 vi, u32 i)         \
 157   { return test_vhash_key_gather (_tm, vi, i, N_KEY_U32); }             \
 158                                                                         \
 159   always_inline u32x4                                                   \
 160   test_vhash_key_gather_4_##N_KEY_U32 (void * _tm, u32 vi, u32 i)       \
 161   {  return test_vhash_4key_gather (_tm, vi, i, N_KEY_U32); }           \
 162                                                                         \
 163   clib_pipeline_stage                                                   \
 164   (test_vhash_gather_keys_stage_##N_KEY_U32,                            \
 165    test_vhash_main_t *, tm, i,                                          \
 166    {                                                                    \
 167      vhash_gather_4key_stage                                            \
 168        (&tm->vhash,                                                     \
 169         /* vector_index */ i,                                           \
 170         test_vhash_key_gather_4_##N_KEY_U32,                            \
 171         tm,                                                             \
 172         N_KEY_U32);                                                     \
 173    })                                                                   \
 174                                                                         \
 175   clib_pipeline_stage_no_inline                                         \
 176   (test_vhash_gather_keys_mod_stage_##N_KEY_U32,                        \
 177    test_vhash_main_t *, tm, i,                                          \
 178    {                                                                    \
 179      vhash_gather_key_stage                                             \
 180        (&tm->vhash,                                                     \
 181         /* vector_index */ tm->n_vectors_div_4,                         \
 182         /* n_vectors */ tm->n_vectors_mod_4,                            \
 183         test_vhash_key_gather_##N_KEY_U32,                              \
 184         tm,                                                             \
 185         N_KEY_U32);                                                     \
 186    })                                                                   \
 187                                                                         \
 188   clib_pipeline_stage                                                   \
 189   (test_vhash_hash_finalize_stage_##N_KEY_U32,                          \
 190    test_vhash_main_t *, tm, i,                                          \
 191    {                                                                    \
 192      vhash_finalize_stage (&tm->vhash, i, N_KEY_U32);                   \
 193    })                                                                   \
 194                                                                         \
 195   clib_pipeline_stage_no_inline                                         \
 196   (test_vhash_hash_finalize_mod_stage_##N_KEY_U32,                      \
 197    test_vhash_main_t *, tm, i,                                          \
 198    {                                                                    \
 199      vhash_finalize_stage (&tm->vhash, tm->n_vectors_div_4, N_KEY_U32); \
 200    })                                                                   \
 201                                                                         \
 202   clib_pipeline_stage                                                   \
 203   (test_vhash_get_stage_##N_KEY_U32,                                    \
 204    test_vhash_main_t *, tm, i,                                          \
 205    {                                                                    \
 206      vhash_get_4_stage (&tm->vhash,                                     \
 207                         /* vector_index */ i,                           \
 208                         test_vhash_get_4result,                         \
 209                         tm, N_KEY_U32);                                 \
 210    })                                                                   \
 211                                                                         \
 212   clib_pipeline_stage_no_inline                                         \
 213   (test_vhash_get_mod_stage_##N_KEY_U32,                                \
 214    test_vhash_main_t *, tm, i,                                          \
 215    {                                                                    \
 216      vhash_get_stage (&tm->vhash,                                       \
 217                       /* vector_index */ tm->n_vectors_div_4,           \
 218                       /* n_vectors */ tm->n_vectors_mod_4,              \
 219                       test_vhash_get_result,                            \
 220                       tm, N_KEY_U32);                                   \
 221    })                                                                   \
 222                                                                         \
 223   clib_pipeline_stage                                                   \
 224   (test_vhash_set_stage_##N_KEY_U32,                                    \
 225    test_vhash_main_t *, tm, i,                                          \
 226    {                                                                    \
 227      vhash_set_stage (&tm->vhash,                                       \
 228                       /* vector_index */ i,                             \
 229                       /* n_vectors */ VECTOR_WORD_TYPE_LEN (u32),       \
 230                       test_vhash_set_result,                            \
 231                       tm, N_KEY_U32);                                   \
 232    })                                                                   \
 233                                                                         \
 234   clib_pipeline_stage_no_inline                                         \
 235   (test_vhash_set_mod_stage_##N_KEY_U32,                                \
 236    test_vhash_main_t *, tm, i,                                          \
 237    {                                                                    \
 238      vhash_set_stage (&tm->vhash,                                       \
 239                       /* vector_index */ tm->n_vectors_div_4,           \
 240                       /* n_vectors */ tm->n_vectors_mod_4,              \
 241                       test_vhash_set_result,                            \
 242                       tm, N_KEY_U32);                                   \
 243    })                                                                   \
 244                                                                         \
 245   clib_pipeline_stage                                                   \
 246   (test_vhash_unset_stage_##N_KEY_U32,                                  \
 247    test_vhash_main_t *, tm, i,                                          \
 248    {                                                                    \
 249      vhash_unset_stage (&tm->vhash,                                     \
 250                         /* vector_index */ i,                           \
 251                         /* n_vectors */ VECTOR_WORD_TYPE_LEN (u32),     \
 252                         test_vhash_unset_result,                        \
 253                         tm, N_KEY_U32);                                 \
 254    })                                                                   \
 255                                                                         \
 256   clib_pipeline_stage_no_inline                                         \
 257   (test_vhash_unset_mod_stage_##N_KEY_U32,                              \
 258    test_vhash_main_t *, tm, i,                                          \
 259    {                                                                    \
 260      vhash_unset_stage (&tm->vhash,                                     \
 261                         /* vector_index */ tm->n_vectors_div_4,         \
 262                         /* n_vectors */ tm->n_vectors_mod_4,            \
 263                         test_vhash_unset_result,                        \
 264                         tm, N_KEY_U32);                                 \
 265    })
 266
 267 _(1);
 268 _(2);
 269 _(3);
 270 _(4);
 271 _(5);
 272 _(6);
 273
 274 #undef _
 275
 276 #define _(N_KEY_U32)                                                    \
 277   clib_pipeline_stage                                                   \
 278   (test_vhash_hash_mix_stage_##N_KEY_U32,                               \
 279    test_vhash_main_t *, tm, i,                                          \
 280    {                                                                    \
 281      vhash_mix_stage (&tm->vhash, i, N_KEY_U32);                        \
 282    })                                                                   \
 283                                                                         \
 284   clib_pipeline_stage_no_inline                                         \
 285   (test_vhash_hash_mix_mod_stage_##N_KEY_U32,                           \
 286    test_vhash_main_t *, tm, i,                                          \
 287    {                                                                    \
 288      vhash_mix_stage (&tm->vhash, tm->n_vectors_div_4, N_KEY_U32);      \
 289    })
 290
 291 _(4);
 292 _(5);
 293 _(6);
 294
 295 #undef _
 296
 297 typedef enum
 298 {
 299   GET, SET, UNSET,
 300 } test_vhash_op_t;
 301
 302 static void
 303 test_vhash_op (test_vhash_main_t * tm,
 304                u32 * key_indices,
 305                u32 * results, uword n_keys, test_vhash_op_t op)
 306 {
 307   vhash_validate_sizes (&tm->vhash, tm->n_key_u32, n_keys);
 308
 309   tm->vhash_results = results;
 310   tm->vhash_key_indices = key_indices;
 311   tm->n_vectors_div_4 = n_keys / 4;
 312   tm->n_vectors_mod_4 = n_keys % 4;
 313
 314   if (tm->n_vectors_div_4 > 0)
 315     {
 316       switch (tm->n_key_u32)
 317         {
 318         default:
 319           ASSERT (0);
 320           break;
 321
 322 #define _(N_KEY_U32)                                            \
 323         case N_KEY_U32:                                         \
 324           if (op == GET)                                        \
 325             clib_pipeline_run_3_stage                           \
 326               (tm->n_vectors_div_4,                             \
 327                tm,                                              \
 328                test_vhash_gather_keys_stage_##N_KEY_U32,        \
 329                test_vhash_hash_finalize_stage_##N_KEY_U32,      \
 330                test_vhash_get_stage_##N_KEY_U32);               \
 331           else if (op == SET)                                   \
 332             clib_pipeline_run_3_stage                           \
 333               (tm->n_vectors_div_4,                             \
 334                tm,                                              \
 335                test_vhash_gather_keys_stage_##N_KEY_U32,        \
 336                test_vhash_hash_finalize_stage_##N_KEY_U32,      \
 337                test_vhash_set_stage_##N_KEY_U32);               \
 338           else                                                  \
 339             clib_pipeline_run_3_stage                           \
 340               (tm->n_vectors_div_4,                             \
 341                tm,                                              \
 342                test_vhash_gather_keys_stage_##N_KEY_U32,        \
 343                test_vhash_hash_finalize_stage_##N_KEY_U32,      \
 344                test_vhash_unset_stage_##N_KEY_U32);             \
 345           break;
 346
 347           _(1);
 348           _(2);
 349           _(3);
 350
 351 #undef _
 352
 353 #define _(N_KEY_U32)                                            \
 354         case N_KEY_U32:                                         \
 355           if (op == GET)                                        \
 356             clib_pipeline_run_4_stage                           \
 357               (tm->n_vectors_div_4,                             \
 358                tm,                                              \
 359                test_vhash_gather_keys_stage_##N_KEY_U32,        \
 360                test_vhash_hash_mix_stage_##N_KEY_U32,           \
 361                test_vhash_hash_finalize_stage_##N_KEY_U32,      \
 362                test_vhash_get_stage_##N_KEY_U32);               \
 363           else if (op == SET)                                   \
 364             clib_pipeline_run_4_stage                           \
 365               (tm->n_vectors_div_4,                             \
 366                tm,                                              \
 367                test_vhash_gather_keys_stage_##N_KEY_U32,        \
 368                test_vhash_hash_mix_stage_##N_KEY_U32,           \
 369                test_vhash_hash_finalize_stage_##N_KEY_U32,      \
 370                test_vhash_set_stage_##N_KEY_U32);               \
 371           else                                                  \
 372             clib_pipeline_run_4_stage                           \
 373               (tm->n_vectors_div_4,                             \
 374                tm,                                              \
 375                test_vhash_gather_keys_stage_##N_KEY_U32,        \
 376                test_vhash_hash_mix_stage_##N_KEY_U32,           \
 377                test_vhash_hash_finalize_stage_##N_KEY_U32,      \
 378                test_vhash_unset_stage_##N_KEY_U32);             \
 379           break;
 380
 381           _(4);
 382           _(5);
 383           _(6);
 384
 385 #undef _
 386         }
 387     }
 388
 389
 390   if (tm->n_vectors_mod_4 > 0)
 391     {
 392       switch (tm->n_key_u32)
 393         {
 394         default:
 395           ASSERT (0);
 396           break;
 397
 398 #define _(N_KEY_U32)                                            \
 399         case N_KEY_U32:                                         \
 400           if (op == GET)                                        \
 401             clib_pipeline_run_3_stage                           \
 402               (1,                                               \
 403                tm,                                              \
 404                test_vhash_gather_keys_mod_stage_##N_KEY_U32,    \
 405                test_vhash_hash_finalize_mod_stage_##N_KEY_U32,  \
 406                test_vhash_get_mod_stage_##N_KEY_U32);           \
 407           else if (op == SET)                                   \
 408             clib_pipeline_run_3_stage                           \
 409               (1,                                               \
 410                tm,                                              \
 411                test_vhash_gather_keys_mod_stage_##N_KEY_U32,    \
 412                test_vhash_hash_finalize_mod_stage_##N_KEY_U32,  \
 413                test_vhash_set_mod_stage_##N_KEY_U32);           \
 414           else                                                  \
 415             clib_pipeline_run_3_stage                           \
 416               (1,                                               \
 417                tm,                                              \
 418                test_vhash_gather_keys_mod_stage_##N_KEY_U32,    \
 419                test_vhash_hash_finalize_mod_stage_##N_KEY_U32,  \
 420                test_vhash_unset_mod_stage_##N_KEY_U32);         \
 421         break;
 422
 423           _(1);
 424           _(2);
 425           _(3);
 426
 427 #undef _
 428
 429 #define _(N_KEY_U32)                                            \
 430         case N_KEY_U32:                                         \
 431           if (op == GET)                                        \
 432             clib_pipeline_run_4_stage                           \
 433               (1,                                               \
 434                tm,                                              \
 435                test_vhash_gather_keys_mod_stage_##N_KEY_U32,    \
 436                test_vhash_hash_mix_mod_stage_##N_KEY_U32,       \
 437                test_vhash_hash_finalize_mod_stage_##N_KEY_U32,  \
 438                test_vhash_get_mod_stage_##N_KEY_U32);           \
 439           else if (op == SET)                                   \
 440             clib_pipeline_run_4_stage                           \
 441               (1,                                               \
 442                tm,                                              \
 443                test_vhash_gather_keys_mod_stage_##N_KEY_U32,    \
 444                test_vhash_hash_mix_mod_stage_##N_KEY_U32,       \
 445                test_vhash_hash_finalize_mod_stage_##N_KEY_U32,  \
 446                test_vhash_set_mod_stage_##N_KEY_U32);           \
 447           else                                                  \
 448             clib_pipeline_run_4_stage                           \
 449               (1,                                               \
 450                tm,                                              \
 451                test_vhash_gather_keys_mod_stage_##N_KEY_U32,    \
 452                test_vhash_hash_mix_mod_stage_##N_KEY_U32,       \
 453                test_vhash_hash_finalize_mod_stage_##N_KEY_U32,  \
 454                test_vhash_unset_mod_stage_##N_KEY_U32);         \
 455           break;
 456
 457           _(4);
 458           _(5);
 459           _(6);
 460
 461 #undef _
 462         }
 463     }
 464 }
 465
 466 int
 467 test_vhash_main (unformat_input_t * input)
 468 {
 469   clib_error_t *error = 0;
 470   test_vhash_main_t _tm, *tm = &_tm;
 471   vhash_t *vh = &tm->vhash;
 472   uword i, j;
 473
 474   memset (tm, 0, sizeof (tm[0]));
 475   tm->n_iter = 100;
 476   tm->seed = 1;
 477   tm->n_keys = 1;
 478   tm->n_key_u32 = 1;
 479   tm->log2_size = 8;
 480   tm->verbose = 0;
 481
 482   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
 483     {
 484       if (unformat (input, "iter %d", &tm->n_iter))
 485         ;
 486       else if (unformat (input, "seed %d", &tm->seed))
 487         ;
 488       else if (unformat (input, "n-keys %d", &tm->n_keys))
 489         ;
 490       else if (unformat (input, "log2-size %d", &tm->log2_size))
 491         ;
 492       else if (unformat (input, "key-words %d", &tm->n_key_u32))
 493         ;
 494       else if (unformat (input, "verbose %=", &tm->verbose, 1))
 495         ;
 496       else
 497         {
 498           error = clib_error_create ("unknown input `%U'\n",
 499                                      format_unformat_error, input);
 500           goto done;
 501         }
 502     }
 503
 504   if (tm->seed == 0)
 505     tm->seed = random_default_seed ();
 506
 507   clib_warning ("iter %d seed %d n-keys %d log2-size %d key-words %d",
 508                 tm->n_iter, tm->seed, tm->n_keys, tm->log2_size,
 509                 tm->n_key_u32);
 510
 511   {
 512     u32 seeds[3];
 513     seeds[0] = seeds[1] = seeds[2] = 0xdeadbeef;
 514     vhash_init (vh, tm->log2_size, tm->n_key_u32, seeds);
 515   }
 516
 517   /* Choose unique keys. */
 518   vec_resize (tm->keys, tm->n_keys * tm->n_key_u32);
 519   vec_resize (tm->key_hash, tm->n_key_u32);
 520   for (i = j = 0; i < vec_len (tm->keys); i++, j++)
 521     {
 522       j = j == tm->n_key_u32 ? 0 : j;
 523       do
 524         {
 525           tm->keys[i] = random_u32 (&tm->seed);
 526         }
 527       while (hash_get (tm->key_hash[j], tm->keys[i]));
 528       hash_set (tm->key_hash[j], tm->keys[i], 0);
 529     }
 530
 531   vec_resize (tm->results, tm->n_keys);
 532   for (i = 0; i < vec_len (tm->results); i++)
 533     {
 534       do
 535         {
 536           tm->results[i] = random_u32 (&tm->seed);
 537         }
 538       while (tm->results[i] == ~0);
 539     }
 540
 541   vec_resize_aligned (tm->vhash_get_results, tm->n_keys,
 542                       CLIB_CACHE_LINE_BYTES);
 543   vec_clone (tm->vhash_get_key_indices, tm->results);
 544   for (i = 0; i < vec_len (tm->vhash_get_key_indices); i++)
 545     tm->vhash_get_key_indices[i] = i;
 546
 547   {
 548     uword *is_set_bitmap = 0;
 549     uword *to_set_bitmap = 0;
 550     uword *to_unset_bitmap = 0;
 551     u32 *to_set = 0, *to_unset = 0;
 552     u32 *to_set_results = 0, *to_unset_results = 0;
 553     u64 t[2];
 554
 555     for (i = 0; i < tm->n_iter; i++)
 556       {
 557         vec_reset_length (to_set);
 558         vec_reset_length (to_unset);
 559         vec_reset_length (to_set_results);
 560         vec_reset_length (to_unset_results);
 561
 562         do
 563           {
 564             to_set_bitmap = clib_bitmap_random (to_set_bitmap,
 565                                                 tm->n_keys, &tm->seed);
 566           }
 567         while (clib_bitmap_is_zero (to_set_bitmap));
 568         to_unset_bitmap = clib_bitmap_dup_and (to_set_bitmap, is_set_bitmap);
 569         to_set_bitmap = clib_bitmap_andnot (to_set_bitmap, to_unset_bitmap);
 570
 571         /* *INDENT-OFF* */
 572         clib_bitmap_foreach (j, to_set_bitmap, ({
 573               vec_add1 (to_set, j);
 574               vec_add1 (to_set_results, tm->results[j]);
 575         }));
 576         /* *INDENT-ON* */
 577         /* *INDENT-OFF* */
 578         clib_bitmap_foreach (j, to_unset_bitmap, ({
 579               vec_add1 (to_unset, j);
 580               vec_add1 (to_unset_results, 0xdeadbeef);
 581         }));
 582         /* *INDENT-ON* */
 583
 584         if (vec_len (to_set) > 0)
 585           {
 586             t[0] = clib_cpu_time_now ();
 587             test_vhash_op (tm, to_set, to_set_results, vec_len (to_set), SET);
 588             t[1] = clib_cpu_time_now ();
 589             tm->set_stats.n_clocks += t[1] - t[0];
 590             tm->set_stats.n_vectors += vec_len (to_set);
 591             tm->set_stats.n_calls += 1;
 592             is_set_bitmap = clib_bitmap_or (is_set_bitmap, to_set_bitmap);
 593           }
 594
 595         t[0] = clib_cpu_time_now ();
 596         test_vhash_op (tm, tm->vhash_get_key_indices,
 597                        tm->vhash_get_results,
 598                        vec_len (tm->vhash_get_key_indices), GET);
 599         t[1] = clib_cpu_time_now ();
 600         tm->get_stats.n_clocks += t[1] - t[0];
 601         tm->get_stats.n_vectors += vec_len (tm->vhash_get_key_indices);
 602         tm->get_stats.n_calls += 1;
 603
 604         for (j = 0; j < vec_len (tm->vhash_get_results); j++)
 605           {
 606             u32 r0 = tm->vhash_get_results[j];
 607             u32 r1 = tm->results[j];
 608             if (clib_bitmap_get (is_set_bitmap, j))
 609               {
 610                 if (r0 != r1)
 611                   os_panic ();
 612               }
 613             else
 614               {
 615                 if (r0 != ~0)
 616                   os_panic ();
 617               }
 618           }
 619
 620         if (vh->n_elts != clib_bitmap_count_set_bits (is_set_bitmap))
 621           os_panic ();
 622
 623         if (vec_len (to_unset) > 0)
 624           {
 625             t[0] = clib_cpu_time_now ();
 626             test_vhash_op (tm, to_unset, to_unset_results,
 627                            vec_len (to_unset), UNSET);
 628             t[1] = clib_cpu_time_now ();
 629             tm->unset_stats.n_clocks += t[1] - t[0];
 630             tm->unset_stats.n_vectors += vec_len (to_unset);
 631             tm->unset_stats.n_calls += 1;
 632             is_set_bitmap =
 633               clib_bitmap_andnot (is_set_bitmap, to_unset_bitmap);
 634           }
 635
 636         t[0] = clib_cpu_time_now ();
 637         test_vhash_op (tm, tm->vhash_get_key_indices,
 638                        tm->vhash_get_results,
 639                        vec_len (tm->vhash_get_key_indices), GET);
 640         t[1] = clib_cpu_time_now ();
 641         tm->get_stats.n_clocks += t[1] - t[0];
 642         tm->get_stats.n_vectors += vec_len (tm->vhash_get_key_indices);
 643         tm->get_stats.n_calls += 1;
 644
 645         for (j = 0; j < vec_len (tm->vhash_get_results); j++)
 646           {
 647             u32 r0 = tm->vhash_get_results[j];
 648             u32 r1 = tm->results[j];
 649             if (clib_bitmap_get (is_set_bitmap, j))
 650               {
 651                 if (r0 != r1)
 652                   os_panic ();
 653               }
 654             else
 655               {
 656                 if (r0 != ~0)
 657                   os_panic ();
 658               }
 659           }
 660
 661         if (vh->n_elts != clib_bitmap_count_set_bits (is_set_bitmap))
 662           os_panic ();
 663       }
 664
 665     vhash_resize (vh, tm->log2_size + 1);
 666
 667     test_vhash_op (tm, tm->vhash_get_key_indices,
 668                    tm->vhash_get_results,
 669                    vec_len (tm->vhash_get_key_indices), GET);
 670
 671     for (j = 0; j < vec_len (tm->vhash_get_results); j++)
 672       {
 673         u32 r0 = tm->vhash_get_results[j];
 674         u32 r1 = tm->results[j];
 675         if (clib_bitmap_get (is_set_bitmap, j))
 676           {
 677             if (r0 != r1)
 678               os_panic ();
 679           }
 680         else
 681           {
 682             if (r0 != ~0)
 683               os_panic ();
 684           }
 685       }
 686
 687     if (vh->n_elts != clib_bitmap_count_set_bits (is_set_bitmap))
 688       os_panic ();
 689   }
 690
 691   {
 692     clib_time_t ct;
 693
 694     clib_time_init (&ct);
 695
 696     clib_warning ("%.4e clocks/get %.4e gets/call %.4e gets/sec",
 697                   (f64) tm->get_stats.n_clocks /
 698                   (f64) tm->get_stats.n_vectors,
 699                   (f64) tm->get_stats.n_vectors / (f64) tm->get_stats.n_calls,
 700                   (f64) tm->get_stats.n_vectors /
 701                   (f64) (tm->get_stats.n_clocks * ct.seconds_per_clock));
 702     if (tm->set_stats.n_calls > 0)
 703       clib_warning ("%.4e clocks/set %.4e sets/call %.4e sets/sec",
 704                     (f64) tm->set_stats.n_clocks /
 705                     (f64) tm->set_stats.n_vectors,
 706                     (f64) tm->set_stats.n_vectors /
 707                     (f64) tm->set_stats.n_calls,
 708                     (f64) tm->set_stats.n_vectors /
 709                     (f64) (tm->set_stats.n_clocks * ct.seconds_per_clock));
 710     if (tm->unset_stats.n_calls > 0)
 711       clib_warning ("%.4e clocks/unset %.4e unsets/call %.4e unsets/sec",
 712                     (f64) tm->unset_stats.n_clocks /
 713                     (f64) tm->unset_stats.n_vectors,
 714                     (f64) tm->unset_stats.n_vectors /
 715                     (f64) tm->unset_stats.n_calls,
 716                     (f64) tm->unset_stats.n_vectors /
 717                     (f64) (tm->unset_stats.n_clocks * ct.seconds_per_clock));
 718   }
 719
 720 done:
 721   if (error)
 722     clib_error_report (error);
 723   return 0;
 724 }
 725
 726 #endif /* CLIB_HAVE_VEC128 */
 727
 728 #ifndef CLIB_HAVE_VEC128
 729 int
 730 test_vhash_main (unformat_input_t * input)
 731 {
 732   clib_error ("compiled without vector support");
 733   return 0;
 734 }
 735 #endif
 736
 737 #ifdef CLIB_UNIX
 738 int
 739 main (int argc, char *argv[])
 740 {
 741   unformat_input_t i;
 742   int r;
 743
 744   unformat_init_command_line (&i, argv);
 745   r = test_vhash_main (&i);
 746   unformat_free (&i);
 747   return r;
 748 }
 749 #endif
 750
 751 /*
 752  * fd.io coding-style-patch-verification: ON
 753  *
 754  * Local Variables:
 755  * eval: (c-set-style "gnu")
 756  * End:
 757  */