test/test/test_hash_readwrite.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <inttypes.h>
   6 #include <locale.h>
   7
   8 #include <rte_cycles.h>
   9 #include <rte_hash.h>
  10 #include <rte_hash_crc.h>
  11 #include <rte_jhash.h>
  12 #include <rte_launch.h>
  13 #include <rte_malloc.h>
  14 #include <rte_random.h>
  15 #include <rte_spinlock.h>
  16
  17 #include "test.h"
  18
  19 #define RTE_RWTEST_FAIL 0
  20
  21 #define TOTAL_ENTRY (16*1024*1024)
  22 #define TOTAL_INSERT (15*1024*1024)
  23
  24 #define NUM_TEST 3
  25 unsigned int core_cnt[NUM_TEST] = {2, 4, 8};
  26
  27 struct perf {
  28         uint32_t single_read;
  29         uint32_t single_write;
  30         uint32_t read_only[NUM_TEST];
  31         uint32_t write_only[NUM_TEST];
  32         uint32_t read_write_r[NUM_TEST];
  33         uint32_t read_write_w[NUM_TEST];
  34 };
  35
  36 static struct perf htm_results, non_htm_results;
  37
  38 struct {
  39         uint32_t *keys;
  40         uint32_t *found;
  41         uint32_t num_insert;
  42         uint32_t rounded_tot_insert;
  43         struct rte_hash *h;
  44 } tbl_rw_test_param;
  45
  46 static rte_atomic64_t gcycles;
  47 static rte_atomic64_t ginsertions;
  48
  49 static rte_atomic64_t gread_cycles;
  50 static rte_atomic64_t gwrite_cycles;
  51
  52 static rte_atomic64_t greads;
  53 static rte_atomic64_t gwrites;
  54
  55 static int
  56 test_hash_readwrite_worker(__attribute__((unused)) void *arg)
  57 {
  58         uint64_t i, offset;
  59         uint32_t lcore_id = rte_lcore_id();
  60         uint64_t begin, cycles;
  61         int ret;
  62
  63         offset = (lcore_id - rte_get_master_lcore())
  64                         * tbl_rw_test_param.num_insert;
  65
  66         printf("Core #%d inserting and reading %d: %'"PRId64" - %'"PRId64"\n",
  67                lcore_id, tbl_rw_test_param.num_insert,
  68                offset, offset + tbl_rw_test_param.num_insert);
  69
  70         begin = rte_rdtsc_precise();
  71
  72         for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
  73
  74                 if (rte_hash_lookup(tbl_rw_test_param.h,
  75                                 tbl_rw_test_param.keys + i) > 0)
  76                         break;
  77
  78                 ret = rte_hash_add_key(tbl_rw_test_param.h,
  79                                      tbl_rw_test_param.keys + i);
  80                 if (ret < 0)
  81                         break;
  82
  83                 if (rte_hash_lookup(tbl_rw_test_param.h,
  84                                 tbl_rw_test_param.keys + i) != ret)
  85                         break;
  86         }
  87
  88         cycles = rte_rdtsc_precise() - begin;
  89         rte_atomic64_add(&gcycles, cycles);
  90         rte_atomic64_add(&ginsertions, i - offset);
  91
  92         for (; i < offset + tbl_rw_test_param.num_insert; i++)
  93                 tbl_rw_test_param.keys[i] = RTE_RWTEST_FAIL;
  94
  95         return 0;
  96 }
  97
  98 static int
  99 init_params(int use_htm, int use_jhash)
 100 {
 101         unsigned int i;
 102
 103         uint32_t *keys = NULL;
 104         uint32_t *found = NULL;
 105         struct rte_hash *handle;
 106
 107         struct rte_hash_parameters hash_params = {
 108                 .entries = TOTAL_ENTRY,
 109                 .key_len = sizeof(uint32_t),
 110                 .hash_func_init_val = 0,
 111                 .socket_id = rte_socket_id(),
 112         };
 113         if (use_jhash)
 114                 hash_params.hash_func = rte_jhash;
 115         else
 116                 hash_params.hash_func = rte_hash_crc;
 117
 118         if (use_htm)
 119                 hash_params.extra_flag =
 120                         RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
 121                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
 122         else
 123                 hash_params.extra_flag =
 124                         RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
 125
 126         hash_params.name = "tests";
 127
 128         handle = rte_hash_create(&hash_params);
 129         if (handle == NULL) {
 130                 printf("hash creation failed");
 131                 return -1;
 132         }
 133
 134         tbl_rw_test_param.h = handle;
 135         keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
 136
 137         if (keys == NULL) {
 138                 printf("RTE_MALLOC failed\n");
 139                 goto err;
 140         }
 141
 142         found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
 143         if (found == NULL) {
 144                 printf("RTE_ZMALLOC failed\n");
 145                 goto err;
 146         }
 147
 148         tbl_rw_test_param.keys = keys;
 149         tbl_rw_test_param.found = found;
 150
 151         for (i = 0; i < TOTAL_ENTRY; i++)
 152                 keys[i] = i;
 153
 154         return 0;
 155
 156 err:
 157         rte_free(keys);
 158         rte_hash_free(handle);
 159
 160         return -1;
 161 }
 162
 163 static int
 164 test_hash_readwrite_functional(int use_htm)
 165 {
 166         unsigned int i;
 167         const void *next_key;
 168         void *next_data;
 169         uint32_t iter = 0;
 170
 171         uint32_t duplicated_keys = 0;
 172         uint32_t lost_keys = 0;
 173         int use_jhash = 1;
 174
 175         rte_atomic64_init(&gcycles);
 176         rte_atomic64_clear(&gcycles);
 177
 178         rte_atomic64_init(&ginsertions);
 179         rte_atomic64_clear(&ginsertions);
 180
 181         if (init_params(use_htm, use_jhash) != 0)
 182                 goto err;
 183
 184         tbl_rw_test_param.num_insert =
 185                 TOTAL_INSERT / rte_lcore_count();
 186
 187         tbl_rw_test_param.rounded_tot_insert =
 188                 tbl_rw_test_param.num_insert
 189                 * rte_lcore_count();
 190
 191         printf("++++++++Start function tests:+++++++++\n");
 192
 193         /* Fire all threads. */
 194         rte_eal_mp_remote_launch(test_hash_readwrite_worker,
 195                                  NULL, CALL_MASTER);
 196         rte_eal_mp_wait_lcore();
 197
 198         while (rte_hash_iterate(tbl_rw_test_param.h, &next_key,
 199                         &next_data, &iter) >= 0) {
 200                 /* Search for the key in the list of keys added .*/
 201                 i = *(const uint32_t *)next_key;
 202                 tbl_rw_test_param.found[i]++;
 203         }
 204
 205         for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
 206                 if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
 207                         if (tbl_rw_test_param.found[i] > 1) {
 208                                 duplicated_keys++;
 209                                 break;
 210                         }
 211                         if (tbl_rw_test_param.found[i] == 0) {
 212                                 lost_keys++;
 213                                 printf("key %d is lost\n", i);
 214                                 break;
 215                         }
 216                 }
 217         }
 218
 219         if (duplicated_keys > 0) {
 220                 printf("%d key duplicated\n", duplicated_keys);
 221                 goto err_free;
 222         }
 223
 224         if (lost_keys > 0) {
 225                 printf("%d key lost\n", lost_keys);
 226                 goto err_free;
 227         }
 228
 229         printf("No key corrupted during read-write test.\n");
 230
 231         unsigned long long int cycles_per_insertion =
 232                 rte_atomic64_read(&gcycles) /
 233                 rte_atomic64_read(&ginsertions);
 234
 235         printf("cycles per insertion and lookup: %llu\n", cycles_per_insertion);
 236
 237         rte_free(tbl_rw_test_param.found);
 238         rte_free(tbl_rw_test_param.keys);
 239         rte_hash_free(tbl_rw_test_param.h);
 240         printf("+++++++++Complete function tests+++++++++\n");
 241         return 0;
 242
 243 err_free:
 244         rte_free(tbl_rw_test_param.found);
 245         rte_free(tbl_rw_test_param.keys);
 246         rte_hash_free(tbl_rw_test_param.h);
 247 err:
 248         return -1;
 249 }
 250
 251 static int
 252 test_rw_reader(__attribute__((unused)) void *arg)
 253 {
 254         uint64_t i;
 255         uint64_t begin, cycles;
 256         uint64_t read_cnt = (uint64_t)((uintptr_t)arg);
 257
 258         begin = rte_rdtsc_precise();
 259         for (i = 0; i < read_cnt; i++) {
 260                 void *data;
 261                 rte_hash_lookup_data(tbl_rw_test_param.h,
 262                                 tbl_rw_test_param.keys + i,
 263                                 &data);
 264                 if (i != (uint64_t)(uintptr_t)data) {
 265                         printf("lookup find wrong value %"PRIu64","
 266                                 "%"PRIu64"\n", i,
 267                                 (uint64_t)(uintptr_t)data);
 268                         break;
 269                 }
 270         }
 271
 272         cycles = rte_rdtsc_precise() - begin;
 273         rte_atomic64_add(&gread_cycles, cycles);
 274         rte_atomic64_add(&greads, i);
 275         return 0;
 276 }
 277
 278 static int
 279 test_rw_writer(__attribute__((unused)) void *arg)
 280 {
 281         uint64_t i;
 282         uint32_t lcore_id = rte_lcore_id();
 283         uint64_t begin, cycles;
 284         int ret;
 285         uint64_t start_coreid = (uint64_t)(uintptr_t)arg;
 286         uint64_t offset;
 287
 288         offset = TOTAL_INSERT / 2 + (lcore_id - start_coreid)
 289                                         * tbl_rw_test_param.num_insert;
 290         begin = rte_rdtsc_precise();
 291         for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
 292                 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
 293                                 tbl_rw_test_param.keys + i,
 294                                 (void *)((uintptr_t)i));
 295                 if (ret < 0) {
 296                         printf("writer failed %"PRIu64"\n", i);
 297                         break;
 298                 }
 299         }
 300
 301         cycles = rte_rdtsc_precise() - begin;
 302         rte_atomic64_add(&gwrite_cycles, cycles);
 303         rte_atomic64_add(&gwrites, tbl_rw_test_param.num_insert);
 304         return 0;
 305 }
 306
 307 static int
 308 test_hash_readwrite_perf(struct perf *perf_results, int use_htm,
 309                                                         int reader_faster)
 310 {
 311         unsigned int n;
 312         int ret;
 313         int start_coreid;
 314         uint64_t i, read_cnt;
 315
 316         const void *next_key;
 317         void *next_data;
 318         uint32_t iter = 0;
 319         int use_jhash = 0;
 320
 321         uint32_t duplicated_keys = 0;
 322         uint32_t lost_keys = 0;
 323
 324         uint64_t start = 0, end = 0;
 325
 326         rte_atomic64_init(&greads);
 327         rte_atomic64_init(&gwrites);
 328         rte_atomic64_clear(&gwrites);
 329         rte_atomic64_clear(&greads);
 330
 331         rte_atomic64_init(&gread_cycles);
 332         rte_atomic64_clear(&gread_cycles);
 333         rte_atomic64_init(&gwrite_cycles);
 334         rte_atomic64_clear(&gwrite_cycles);
 335
 336         if (init_params(use_htm, use_jhash) != 0)
 337                 goto err;
 338
 339         /*
 340          * Do a readers finish faster or writers finish faster test.
 341          * When readers finish faster, we timing the readers, and when writers
 342          * finish faster, we timing the writers.
 343          * Divided by 10 or 2 is just experimental values to vary the workload
 344          * of readers.
 345          */
 346         if (reader_faster) {
 347                 printf("++++++Start perf test: reader++++++++\n");
 348                 read_cnt = TOTAL_INSERT / 10;
 349         } else {
 350                 printf("++++++Start perf test: writer++++++++\n");
 351                 read_cnt = TOTAL_INSERT / 2;
 352         }
 353
 354         /* We first test single thread performance */
 355         start = rte_rdtsc_precise();
 356         /* Insert half of the keys */
 357         for (i = 0; i < TOTAL_INSERT / 2; i++) {
 358                 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
 359                                      tbl_rw_test_param.keys + i,
 360                                         (void *)((uintptr_t)i));
 361                 if (ret < 0) {
 362                         printf("Failed to insert half of keys\n");
 363                         goto err_free;
 364                 }
 365         }
 366         end = rte_rdtsc_precise() - start;
 367         perf_results->single_write = end / i;
 368
 369         start = rte_rdtsc_precise();
 370
 371         for (i = 0; i < read_cnt; i++) {
 372                 void *data;
 373                 rte_hash_lookup_data(tbl_rw_test_param.h,
 374                                 tbl_rw_test_param.keys + i,
 375                                 &data);
 376                 if (i != (uint64_t)(uintptr_t)data) {
 377                         printf("lookup find wrong value"
 378                                         " %"PRIu64",%"PRIu64"\n", i,
 379                                         (uint64_t)(uintptr_t)data);
 380                         break;
 381                 }
 382         }
 383         end = rte_rdtsc_precise() - start;
 384         perf_results->single_read = end / i;
 385
 386         for (n = 0; n < NUM_TEST; n++) {
 387                 unsigned int tot_lcore = rte_lcore_count();
 388                 if (tot_lcore < core_cnt[n] * 2 + 1)
 389                         goto finish;
 390
 391                 rte_atomic64_clear(&greads);
 392                 rte_atomic64_clear(&gread_cycles);
 393                 rte_atomic64_clear(&gwrites);
 394                 rte_atomic64_clear(&gwrite_cycles);
 395
 396                 rte_hash_reset(tbl_rw_test_param.h);
 397
 398                 tbl_rw_test_param.num_insert = TOTAL_INSERT / 2 / core_cnt[n];
 399                 tbl_rw_test_param.rounded_tot_insert = TOTAL_INSERT / 2 +
 400                                                 tbl_rw_test_param.num_insert *
 401                                                 core_cnt[n];
 402
 403                 for (i = 0; i < TOTAL_INSERT / 2; i++) {
 404                         ret = rte_hash_add_key_data(tbl_rw_test_param.h,
 405                                         tbl_rw_test_param.keys + i,
 406                                         (void *)((uintptr_t)i));
 407                         if (ret < 0) {
 408                                 printf("Failed to insert half of keys\n");
 409                                 goto err_free;
 410                         }
 411                 }
 412
 413                 /* Then test multiple thread case but only all reads or
 414                  * all writes
 415                  */
 416
 417                 /* Test only reader cases */
 418                 for (i = 1; i <= core_cnt[n]; i++)
 419                         rte_eal_remote_launch(test_rw_reader,
 420                                         (void *)(uintptr_t)read_cnt, i);
 421
 422                 rte_eal_mp_wait_lcore();
 423
 424                 start_coreid = i;
 425                 /* Test only writer cases */
 426                 for (; i <= core_cnt[n] * 2; i++)
 427                         rte_eal_remote_launch(test_rw_writer,
 428                                         (void *)((uintptr_t)start_coreid), i);
 429
 430                 rte_eal_mp_wait_lcore();
 431
 432                 if (reader_faster) {
 433                         unsigned long long int cycles_per_insertion =
 434                                 rte_atomic64_read(&gread_cycles) /
 435                                 rte_atomic64_read(&greads);
 436                         perf_results->read_only[n] = cycles_per_insertion;
 437                         printf("Reader only: cycles per lookup: %llu\n",
 438                                                         cycles_per_insertion);
 439                 }
 440
 441                 else {
 442                         unsigned long long int cycles_per_insertion =
 443                                 rte_atomic64_read(&gwrite_cycles) /
 444                                 rte_atomic64_read(&gwrites);
 445                         perf_results->write_only[n] = cycles_per_insertion;
 446                         printf("Writer only: cycles per writes: %llu\n",
 447                                                         cycles_per_insertion);
 448                 }
 449
 450                 rte_atomic64_clear(&greads);
 451                 rte_atomic64_clear(&gread_cycles);
 452                 rte_atomic64_clear(&gwrites);
 453                 rte_atomic64_clear(&gwrite_cycles);
 454
 455                 rte_hash_reset(tbl_rw_test_param.h);
 456
 457                 for (i = 0; i < TOTAL_INSERT / 2; i++) {
 458                         ret = rte_hash_add_key_data(tbl_rw_test_param.h,
 459                                         tbl_rw_test_param.keys + i,
 460                                         (void *)((uintptr_t)i));
 461                         if (ret < 0) {
 462                                 printf("Failed to insert half of keys\n");
 463                                 goto err_free;
 464                         }
 465                 }
 466
 467                 start_coreid = core_cnt[n] + 1;
 468
 469                 if (reader_faster) {
 470                         for (i = core_cnt[n] + 1; i <= core_cnt[n] * 2; i++)
 471                                 rte_eal_remote_launch(test_rw_writer,
 472                                         (void *)((uintptr_t)start_coreid), i);
 473                         for (i = 1; i <= core_cnt[n]; i++)
 474                                 rte_eal_remote_launch(test_rw_reader,
 475                                         (void *)(uintptr_t)read_cnt, i);
 476                 } else {
 477                         for (i = 1; i <= core_cnt[n]; i++)
 478                                 rte_eal_remote_launch(test_rw_reader,
 479                                         (void *)(uintptr_t)read_cnt, i);
 480                         for (; i <= core_cnt[n] * 2; i++)
 481                                 rte_eal_remote_launch(test_rw_writer,
 482                                         (void *)((uintptr_t)start_coreid), i);
 483                 }
 484
 485                 rte_eal_mp_wait_lcore();
 486
 487                 while (rte_hash_iterate(tbl_rw_test_param.h,
 488                                 &next_key, &next_data, &iter) >= 0) {
 489                         /* Search for the key in the list of keys added .*/
 490                         i = *(const uint32_t *)next_key;
 491                         tbl_rw_test_param.found[i]++;
 492                 }
 493
 494                 for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
 495                         if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
 496                                 if (tbl_rw_test_param.found[i] > 1) {
 497                                         duplicated_keys++;
 498                                         break;
 499                                 }
 500                                 if (tbl_rw_test_param.found[i] == 0) {
 501                                         lost_keys++;
 502                                         printf("key %"PRIu64" is lost\n", i);
 503                                         break;
 504                                 }
 505                         }
 506                 }
 507
 508                 if (duplicated_keys > 0) {
 509                         printf("%d key duplicated\n", duplicated_keys);
 510                         goto err_free;
 511                 }
 512
 513                 if (lost_keys > 0) {
 514                         printf("%d key lost\n", lost_keys);
 515                         goto err_free;
 516                 }
 517
 518                 printf("No key corrupted during read-write test.\n");
 519
 520                 if (reader_faster) {
 521                         unsigned long long int cycles_per_insertion =
 522                                 rte_atomic64_read(&gread_cycles) /
 523                                 rte_atomic64_read(&greads);
 524                         perf_results->read_write_r[n] = cycles_per_insertion;
 525                         printf("Read-write cycles per lookup: %llu\n",
 526                                                         cycles_per_insertion);
 527                 }
 528
 529                 else {
 530                         unsigned long long int cycles_per_insertion =
 531                                 rte_atomic64_read(&gwrite_cycles) /
 532                                 rte_atomic64_read(&gwrites);
 533                         perf_results->read_write_w[n] = cycles_per_insertion;
 534                         printf("Read-write cycles per writes: %llu\n",
 535                                                         cycles_per_insertion);
 536                 }
 537         }
 538
 539 finish:
 540         rte_free(tbl_rw_test_param.found);
 541         rte_free(tbl_rw_test_param.keys);
 542         rte_hash_free(tbl_rw_test_param.h);
 543         return 0;
 544
 545 err_free:
 546         rte_free(tbl_rw_test_param.found);
 547         rte_free(tbl_rw_test_param.keys);
 548         rte_hash_free(tbl_rw_test_param.h);
 549
 550 err:
 551         return -1;
 552 }
 553
 554 static int
 555 test_hash_readwrite_main(void)
 556 {
 557         /*
 558          * Variables used to choose different tests.
 559          * use_htm indicates if hardware transactional memory should be used.
 560          * reader_faster indicates if the reader threads should finish earlier
 561          * than writer threads. This is to timing either reader threads or
 562          * writer threads for performance numbers.
 563          */
 564         int use_htm, reader_faster;
 565
 566         if (rte_lcore_count() == 1) {
 567                 printf("More than one lcore is required "
 568                         "to do read write test\n");
 569                 return 0;
 570         }
 571
 572         setlocale(LC_NUMERIC, "");
 573
 574         if (rte_tm_supported()) {
 575                 printf("Hardware transactional memory (lock elision) "
 576                         "is supported\n");
 577
 578                 printf("Test read-write with Hardware transactional memory\n");
 579
 580                 use_htm = 1;
 581                 if (test_hash_readwrite_functional(use_htm) < 0)
 582                         return -1;
 583
 584                 reader_faster = 1;
 585                 if (test_hash_readwrite_perf(&htm_results, use_htm,
 586                                                         reader_faster) < 0)
 587                         return -1;
 588
 589                 reader_faster = 0;
 590                 if (test_hash_readwrite_perf(&htm_results, use_htm,
 591                                                         reader_faster) < 0)
 592                         return -1;
 593         } else {
 594                 printf("Hardware transactional memory (lock elision) "
 595                         "is NOT supported\n");
 596         }
 597
 598         printf("Test read-write without Hardware transactional memory\n");
 599         use_htm = 0;
 600         if (test_hash_readwrite_functional(use_htm) < 0)
 601                 return -1;
 602         reader_faster = 1;
 603         if (test_hash_readwrite_perf(&non_htm_results, use_htm,
 604                                                         reader_faster) < 0)
 605                 return -1;
 606         reader_faster = 0;
 607         if (test_hash_readwrite_perf(&non_htm_results, use_htm,
 608                                                         reader_faster) < 0)
 609                 return -1;
 610
 611         printf("Results summary:\n");
 612
 613         int i;
 614
 615         printf("single read: %u\n", htm_results.single_read);
 616         printf("single write: %u\n", htm_results.single_write);
 617         for (i = 0; i < NUM_TEST; i++) {
 618                 printf("core_cnt: %u\n", core_cnt[i]);
 619                 printf("HTM:\n");
 620                 printf("read only: %u\n", htm_results.read_only[i]);
 621                 printf("write only: %u\n", htm_results.write_only[i]);
 622                 printf("read-write read: %u\n", htm_results.read_write_r[i]);
 623                 printf("read-write write: %u\n", htm_results.read_write_w[i]);
 624
 625                 printf("non HTM:\n");
 626                 printf("read only: %u\n", non_htm_results.read_only[i]);
 627                 printf("write only: %u\n", non_htm_results.write_only[i]);
 628                 printf("read-write read: %u\n",
 629                         non_htm_results.read_write_r[i]);
 630                 printf("read-write write: %u\n",
 631                         non_htm_results.read_write_w[i]);
 632         }
 633
 634         return 0;
 635 }
 636
 637 REGISTER_TEST_COMMAND(hash_readwrite_autotest, test_hash_readwrite_main);