app/test-bbdev/test_bbdev_perf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7
   8 #include <rte_eal.h>
   9 #include <rte_common.h>
  10 #include <rte_dev.h>
  11 #include <rte_launch.h>
  12 #include <rte_bbdev.h>
  13 #include <rte_cycles.h>
  14 #include <rte_lcore.h>
  15 #include <rte_malloc.h>
  16 #include <rte_random.h>
  17 #include <rte_hexdump.h>
  18
  19 #include "main.h"
  20 #include "test_bbdev_vector.h"
  21
  22 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id))
  23
  24 #define MAX_QUEUES RTE_MAX_LCORE
  25
  26 #define OPS_CACHE_SIZE 256U
  27 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
  28
  29 #define SYNC_WAIT 0
  30 #define SYNC_START 1
  31
  32 #define INVALID_QUEUE_ID -1
  33
  34 static struct test_bbdev_vector test_vector;
  35
  36 /* Switch between PMD and Interrupt for throughput TC */
  37 static bool intr_enabled;
  38
  39 /* Represents tested active devices */
  40 static struct active_device {
  41         const char *driver_name;
  42         uint8_t dev_id;
  43         uint16_t supported_ops;
  44         uint16_t queue_ids[MAX_QUEUES];
  45         uint16_t nb_queues;
  46         struct rte_mempool *ops_mempool;
  47         struct rte_mempool *in_mbuf_pool;
  48         struct rte_mempool *hard_out_mbuf_pool;
  49         struct rte_mempool *soft_out_mbuf_pool;
  50 } active_devs[RTE_BBDEV_MAX_DEVS];
  51
  52 static uint8_t nb_active_devs;
  53
  54 /* Data buffers used by BBDEV ops */
  55 struct test_buffers {
  56         struct rte_bbdev_op_data *inputs;
  57         struct rte_bbdev_op_data *hard_outputs;
  58         struct rte_bbdev_op_data *soft_outputs;
  59 };
  60
  61 /* Operation parameters specific for given test case */
  62 struct test_op_params {
  63         struct rte_mempool *mp;
  64         struct rte_bbdev_dec_op *ref_dec_op;
  65         struct rte_bbdev_enc_op *ref_enc_op;
  66         uint16_t burst_sz;
  67         uint16_t num_to_process;
  68         uint16_t num_lcores;
  69         int vector_mask;
  70         rte_atomic16_t sync;
  71         struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES];
  72 };
  73
  74 /* Contains per lcore params */
  75 struct thread_params {
  76         uint8_t dev_id;
  77         uint16_t queue_id;
  78         uint64_t start_time;
  79         double mops;
  80         double mbps;
  81         rte_atomic16_t nb_dequeued;
  82         rte_atomic16_t processing_status;
  83         struct test_op_params *op_params;
  84 };
  85
  86 typedef int (test_case_function)(struct active_device *ad,
  87                 struct test_op_params *op_params);
  88
  89 static inline void
  90 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
  91 {
  92         ad->supported_ops |= (1 << op_type);
  93 }
  94
  95 static inline bool
  96 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type)
  97 {
  98         return ad->supported_ops & (1 << op_type);
  99 }
 100
 101 static inline bool
 102 flags_match(uint32_t flags_req, uint32_t flags_present)
 103 {
 104         return (flags_req & flags_present) == flags_req;
 105 }
 106
 107 static void
 108 clear_soft_out_cap(uint32_t *op_flags)
 109 {
 110         *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT;
 111         *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT;
 112         *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT;
 113 }
 114
 115 static int
 116 check_dev_cap(const struct rte_bbdev_info *dev_info)
 117 {
 118         unsigned int i;
 119         unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs;
 120         const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities;
 121
 122         nb_inputs = test_vector.entries[DATA_INPUT].nb_segments;
 123         nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments;
 124         nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments;
 125
 126         for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) {
 127                 if (op_cap->type != test_vector.op_type)
 128                         continue;
 129
 130                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
 131                         const struct rte_bbdev_op_cap_turbo_dec *cap =
 132                                         &op_cap->cap.turbo_dec;
 133                         /* Ignore lack of soft output capability, just skip
 134                          * checking if soft output is valid.
 135                          */
 136                         if ((test_vector.turbo_dec.op_flags &
 137                                         RTE_BBDEV_TURBO_SOFT_OUTPUT) &&
 138                                         !(cap->capability_flags &
 139                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 140                                 printf(
 141                                         "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n",
 142                                         dev_info->dev_name);
 143                                 clear_soft_out_cap(
 144                                         &test_vector.turbo_dec.op_flags);
 145                         }
 146
 147                         if (!flags_match(test_vector.turbo_dec.op_flags,
 148                                         cap->capability_flags))
 149                                 return TEST_FAILED;
 150                         if (nb_inputs > cap->num_buffers_src) {
 151                                 printf("Too many inputs defined: %u, max: %u\n",
 152                                         nb_inputs, cap->num_buffers_src);
 153                                 return TEST_FAILED;
 154                         }
 155                         if (nb_soft_outputs > cap->num_buffers_soft_out &&
 156                                         (test_vector.turbo_dec.op_flags &
 157                                         RTE_BBDEV_TURBO_SOFT_OUTPUT)) {
 158                                 printf(
 159                                         "Too many soft outputs defined: %u, max: %u\n",
 160                                                 nb_soft_outputs,
 161                                                 cap->num_buffers_soft_out);
 162                                 return TEST_FAILED;
 163                         }
 164                         if (nb_hard_outputs > cap->num_buffers_hard_out) {
 165                                 printf(
 166                                         "Too many hard outputs defined: %u, max: %u\n",
 167                                                 nb_hard_outputs,
 168                                                 cap->num_buffers_hard_out);
 169                                 return TEST_FAILED;
 170                         }
 171                         if (intr_enabled && !(cap->capability_flags &
 172                                         RTE_BBDEV_TURBO_DEC_INTERRUPTS)) {
 173                                 printf(
 174                                         "Dequeue interrupts are not supported!\n");
 175                                 return TEST_FAILED;
 176                         }
 177
 178                         return TEST_SUCCESS;
 179                 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
 180                         const struct rte_bbdev_op_cap_turbo_enc *cap =
 181                                         &op_cap->cap.turbo_enc;
 182
 183                         if (!flags_match(test_vector.turbo_enc.op_flags,
 184                                         cap->capability_flags))
 185                                 return TEST_FAILED;
 186                         if (nb_inputs > cap->num_buffers_src) {
 187                                 printf("Too many inputs defined: %u, max: %u\n",
 188                                         nb_inputs, cap->num_buffers_src);
 189                                 return TEST_FAILED;
 190                         }
 191                         if (nb_hard_outputs > cap->num_buffers_dst) {
 192                                 printf(
 193                                         "Too many hard outputs defined: %u, max: %u\n",
 194                                         nb_hard_outputs, cap->num_buffers_src);
 195                                 return TEST_FAILED;
 196                         }
 197                         if (intr_enabled && !(cap->capability_flags &
 198                                         RTE_BBDEV_TURBO_ENC_INTERRUPTS)) {
 199                                 printf(
 200                                         "Dequeue interrupts are not supported!\n");
 201                                 return TEST_FAILED;
 202                         }
 203
 204                         return TEST_SUCCESS;
 205                 }
 206         }
 207
 208         if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE))
 209                 return TEST_SUCCESS; /* Special case for NULL device */
 210
 211         return TEST_FAILED;
 212 }
 213
 214 /* calculates optimal mempool size not smaller than the val */
 215 static unsigned int
 216 optimal_mempool_size(unsigned int val)
 217 {
 218         return rte_align32pow2(val + 1) - 1;
 219 }
 220
 221 /* allocates mbuf mempool for inputs and outputs */
 222 static struct rte_mempool *
 223 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id,
 224                 int socket_id, unsigned int mbuf_pool_size,
 225                 const char *op_type_str)
 226 {
 227         unsigned int i;
 228         uint32_t max_seg_sz = 0;
 229         char pool_name[RTE_MEMPOOL_NAMESIZE];
 230
 231         /* find max input segment size */
 232         for (i = 0; i < entries->nb_segments; ++i)
 233                 if (entries->segments[i].length > max_seg_sz)
 234                         max_seg_sz = entries->segments[i].length;
 235
 236         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 237                         dev_id);
 238         return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0,
 239                         RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM,
 240                         (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id);
 241 }
 242
 243 static int
 244 create_mempools(struct active_device *ad, int socket_id,
 245                 enum rte_bbdev_op_type op_type, uint16_t num_ops)
 246 {
 247         struct rte_mempool *mp;
 248         unsigned int ops_pool_size, mbuf_pool_size = 0;
 249         char pool_name[RTE_MEMPOOL_NAMESIZE];
 250         const char *op_type_str;
 251
 252         struct op_data_entries *in = &test_vector.entries[DATA_INPUT];
 253         struct op_data_entries *hard_out =
 254                         &test_vector.entries[DATA_HARD_OUTPUT];
 255         struct op_data_entries *soft_out =
 256                         &test_vector.entries[DATA_SOFT_OUTPUT];
 257
 258         /* allocate ops mempool */
 259         ops_pool_size = optimal_mempool_size(RTE_MAX(
 260                         /* Ops used plus 1 reference op */
 261                         RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1),
 262                         /* Minimal cache size plus 1 reference op */
 263                         (unsigned int)(1.5 * rte_lcore_count() *
 264                                         OPS_CACHE_SIZE + 1)),
 265                         OPS_POOL_SIZE_MIN));
 266
 267         op_type_str = rte_bbdev_op_type_str(op_type);
 268         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
 269
 270         snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str,
 271                         ad->dev_id);
 272         mp = rte_bbdev_op_pool_create(pool_name, op_type,
 273                         ops_pool_size, OPS_CACHE_SIZE, socket_id);
 274         TEST_ASSERT_NOT_NULL(mp,
 275                         "ERROR Failed to create %u items ops pool for dev %u on socket %u.",
 276                         ops_pool_size,
 277                         ad->dev_id,
 278                         socket_id);
 279         ad->ops_mempool = mp;
 280
 281         /* Inputs */
 282         mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments);
 283         mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in");
 284         TEST_ASSERT_NOT_NULL(mp,
 285                         "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.",
 286                         mbuf_pool_size,
 287                         ad->dev_id,
 288                         socket_id);
 289         ad->in_mbuf_pool = mp;
 290
 291         /* Hard outputs */
 292         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 293                         hard_out->nb_segments);
 294         mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size,
 295                         "hard_out");
 296         TEST_ASSERT_NOT_NULL(mp,
 297                         "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.",
 298                         mbuf_pool_size,
 299                         ad->dev_id,
 300                         socket_id);
 301         ad->hard_out_mbuf_pool = mp;
 302
 303         if (soft_out->nb_segments == 0)
 304                 return TEST_SUCCESS;
 305
 306         /* Soft outputs */
 307         mbuf_pool_size = optimal_mempool_size(ops_pool_size *
 308                         soft_out->nb_segments);
 309         mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size,
 310                         "soft_out");
 311         TEST_ASSERT_NOT_NULL(mp,
 312                         "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.",
 313                         mbuf_pool_size,
 314                         ad->dev_id,
 315                         socket_id);
 316         ad->soft_out_mbuf_pool = mp;
 317
 318         return 0;
 319 }
 320
 321 static int
 322 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info,
 323                 struct test_bbdev_vector *vector)
 324 {
 325         int ret;
 326         unsigned int queue_id;
 327         struct rte_bbdev_queue_conf qconf;
 328         struct active_device *ad = &active_devs[nb_active_devs];
 329         unsigned int nb_queues;
 330         enum rte_bbdev_op_type op_type = vector->op_type;
 331
 332         nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 333         /* setup device */
 334         ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id);
 335         if (ret < 0) {
 336                 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n",
 337                                 dev_id, nb_queues, info->socket_id, ret);
 338                 return TEST_FAILED;
 339         }
 340
 341         /* configure interrupts if needed */
 342         if (intr_enabled) {
 343                 ret = rte_bbdev_intr_enable(dev_id);
 344                 if (ret < 0) {
 345                         printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id,
 346                                         ret);
 347                         return TEST_FAILED;
 348                 }
 349         }
 350
 351         /* setup device queues */
 352         qconf.socket = info->socket_id;
 353         qconf.queue_size = info->drv.default_queue_conf.queue_size;
 354         qconf.priority = 0;
 355         qconf.deferred_start = 0;
 356         qconf.op_type = op_type;
 357
 358         for (queue_id = 0; queue_id < nb_queues; ++queue_id) {
 359                 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf);
 360                 if (ret != 0) {
 361                         printf(
 362                                         "Allocated all queues (id=%u) at prio%u on dev%u\n",
 363                                         queue_id, qconf.priority, dev_id);
 364                         qconf.priority++;
 365                         ret = rte_bbdev_queue_configure(ad->dev_id, queue_id,
 366                                         &qconf);
 367                 }
 368                 if (ret != 0) {
 369                         printf("All queues on dev %u allocated: %u\n",
 370                                         dev_id, queue_id);
 371                         break;
 372                 }
 373                 ad->queue_ids[queue_id] = queue_id;
 374         }
 375         TEST_ASSERT(queue_id != 0,
 376                         "ERROR Failed to configure any queues on dev %u",
 377                         dev_id);
 378         ad->nb_queues = queue_id;
 379
 380         set_avail_op(ad, op_type);
 381
 382         return TEST_SUCCESS;
 383 }
 384
 385 static int
 386 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info,
 387                 struct test_bbdev_vector *vector)
 388 {
 389         int ret;
 390
 391         active_devs[nb_active_devs].driver_name = info->drv.driver_name;
 392         active_devs[nb_active_devs].dev_id = dev_id;
 393
 394         ret = add_bbdev_dev(dev_id, info, vector);
 395         if (ret == TEST_SUCCESS)
 396                 ++nb_active_devs;
 397         return ret;
 398 }
 399
 400 static uint8_t
 401 populate_active_devices(void)
 402 {
 403         int ret;
 404         uint8_t dev_id;
 405         uint8_t nb_devs_added = 0;
 406         struct rte_bbdev_info info;
 407
 408         RTE_BBDEV_FOREACH(dev_id) {
 409                 rte_bbdev_info_get(dev_id, &info);
 410
 411                 if (check_dev_cap(&info)) {
 412                         printf(
 413                                 "Device %d (%s) does not support specified capabilities\n",
 414                                         dev_id, info.dev_name);
 415                         continue;
 416                 }
 417
 418                 ret = add_active_device(dev_id, &info, &test_vector);
 419                 if (ret != 0) {
 420                         printf("Adding active bbdev %s skipped\n",
 421                                         info.dev_name);
 422                         continue;
 423                 }
 424                 nb_devs_added++;
 425         }
 426
 427         return nb_devs_added;
 428 }
 429
 430 static int
 431 read_test_vector(void)
 432 {
 433         int ret;
 434
 435         memset(&test_vector, 0, sizeof(test_vector));
 436         printf("Test vector file = %s\n", get_vector_filename());
 437         ret = test_bbdev_vector_read(get_vector_filename(), &test_vector);
 438         TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n",
 439                         get_vector_filename());
 440
 441         return TEST_SUCCESS;
 442 }
 443
 444 static int
 445 testsuite_setup(void)
 446 {
 447         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 448
 449         if (populate_active_devices() == 0) {
 450                 printf("No suitable devices found!\n");
 451                 return TEST_SKIPPED;
 452         }
 453
 454         return TEST_SUCCESS;
 455 }
 456
 457 static int
 458 interrupt_testsuite_setup(void)
 459 {
 460         TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n");
 461
 462         /* Enable interrupts */
 463         intr_enabled = true;
 464
 465         /* Special case for NULL device (RTE_BBDEV_OP_NONE) */
 466         if (populate_active_devices() == 0 ||
 467                         test_vector.op_type == RTE_BBDEV_OP_NONE) {
 468                 intr_enabled = false;
 469                 printf("No suitable devices found!\n");
 470                 return TEST_SKIPPED;
 471         }
 472
 473         return TEST_SUCCESS;
 474 }
 475
 476 static void
 477 testsuite_teardown(void)
 478 {
 479         uint8_t dev_id;
 480
 481         /* Unconfigure devices */
 482         RTE_BBDEV_FOREACH(dev_id)
 483                 rte_bbdev_close(dev_id);
 484
 485         /* Clear active devices structs. */
 486         memset(active_devs, 0, sizeof(active_devs));
 487         nb_active_devs = 0;
 488 }
 489
 490 static int
 491 ut_setup(void)
 492 {
 493         uint8_t i, dev_id;
 494
 495         for (i = 0; i < nb_active_devs; i++) {
 496                 dev_id = active_devs[i].dev_id;
 497                 /* reset bbdev stats */
 498                 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id),
 499                                 "Failed to reset stats of bbdev %u", dev_id);
 500                 /* start the device */
 501                 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id),
 502                                 "Failed to start bbdev %u", dev_id);
 503         }
 504
 505         return TEST_SUCCESS;
 506 }
 507
 508 static void
 509 ut_teardown(void)
 510 {
 511         uint8_t i, dev_id;
 512         struct rte_bbdev_stats stats;
 513
 514         for (i = 0; i < nb_active_devs; i++) {
 515                 dev_id = active_devs[i].dev_id;
 516                 /* read stats and print */
 517                 rte_bbdev_stats_get(dev_id, &stats);
 518                 /* Stop the device */
 519                 rte_bbdev_stop(dev_id);
 520         }
 521 }
 522
 523 static int
 524 init_op_data_objs(struct rte_bbdev_op_data *bufs,
 525                 struct op_data_entries *ref_entries,
 526                 struct rte_mempool *mbuf_pool, const uint16_t n,
 527                 enum op_data_type op_type, uint16_t min_alignment)
 528 {
 529         int ret;
 530         unsigned int i, j;
 531
 532         for (i = 0; i < n; ++i) {
 533                 char *data;
 534                 struct op_data_buf *seg = &ref_entries->segments[0];
 535                 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
 536                 TEST_ASSERT_NOT_NULL(m_head,
 537                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 538                                 op_type, n * ref_entries->nb_segments,
 539                                 mbuf_pool->size);
 540
 541                 bufs[i].data = m_head;
 542                 bufs[i].offset = 0;
 543                 bufs[i].length = 0;
 544
 545                 if (op_type == DATA_INPUT) {
 546                         data = rte_pktmbuf_append(m_head, seg->length);
 547                         TEST_ASSERT_NOT_NULL(data,
 548                                         "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 549                                         seg->length, op_type);
 550
 551                         TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment),
 552                                         "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 553                                         data, min_alignment);
 554                         rte_memcpy(data, seg->addr, seg->length);
 555                         bufs[i].length += seg->length;
 556
 557
 558                         for (j = 1; j < ref_entries->nb_segments; ++j) {
 559                                 struct rte_mbuf *m_tail =
 560                                                 rte_pktmbuf_alloc(mbuf_pool);
 561                                 TEST_ASSERT_NOT_NULL(m_tail,
 562                                                 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
 563                                                 op_type,
 564                                                 n * ref_entries->nb_segments,
 565                                                 mbuf_pool->size);
 566                                 seg += 1;
 567
 568                                 data = rte_pktmbuf_append(m_tail, seg->length);
 569                                 TEST_ASSERT_NOT_NULL(data,
 570                                                 "Couldn't append %u bytes to mbuf from %d data type mbuf pool",
 571                                                 seg->length, op_type);
 572
 573                                 TEST_ASSERT(data == RTE_PTR_ALIGN(data,
 574                                                 min_alignment),
 575                                                 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)",
 576                                                 data, min_alignment);
 577                                 rte_memcpy(data, seg->addr, seg->length);
 578                                 bufs[i].length += seg->length;
 579
 580                                 ret = rte_pktmbuf_chain(m_head, m_tail);
 581                                 TEST_ASSERT_SUCCESS(ret,
 582                                                 "Couldn't chain mbufs from %d data type mbuf pool",
 583                                                 op_type);
 584                         }
 585                 }
 586         }
 587
 588         return 0;
 589 }
 590
 591 static int
 592 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len,
 593                 const int socket)
 594 {
 595         int i;
 596
 597         *buffers = rte_zmalloc_socket(NULL, len, 0, socket);
 598         if (*buffers == NULL) {
 599                 printf("WARNING: Failed to allocate op_data on socket %d\n",
 600                                 socket);
 601                 /* try to allocate memory on other detected sockets */
 602                 for (i = 0; i < socket; i++) {
 603                         *buffers = rte_zmalloc_socket(NULL, len, 0, i);
 604                         if (*buffers != NULL)
 605                                 break;
 606                 }
 607         }
 608
 609         return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS;
 610 }
 611
 612 static int
 613 fill_queue_buffers(struct test_op_params *op_params,
 614                 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp,
 615                 struct rte_mempool *soft_out_mp, uint16_t queue_id,
 616                 uint16_t min_alignment, const int socket_id)
 617 {
 618         int ret;
 619         enum op_data_type type;
 620         const uint16_t n = op_params->num_to_process;
 621
 622         struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
 623                 in_mp,
 624                 soft_out_mp,
 625                 hard_out_mp,
 626         };
 627
 628         struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
 629                 &op_params->q_bufs[socket_id][queue_id].inputs,
 630                 &op_params->q_bufs[socket_id][queue_id].soft_outputs,
 631                 &op_params->q_bufs[socket_id][queue_id].hard_outputs,
 632         };
 633
 634         for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) {
 635                 struct op_data_entries *ref_entries =
 636                                 &test_vector.entries[type];
 637                 if (ref_entries->nb_segments == 0)
 638                         continue;
 639
 640                 ret = allocate_buffers_on_socket(queue_ops[type],
 641                                 n * sizeof(struct rte_bbdev_op_data),
 642                                 socket_id);
 643                 TEST_ASSERT_SUCCESS(ret,
 644                                 "Couldn't allocate memory for rte_bbdev_op_data structs");
 645
 646                 ret = init_op_data_objs(*queue_ops[type], ref_entries,
 647                                 mbuf_pools[type], n, type, min_alignment);
 648                 TEST_ASSERT_SUCCESS(ret,
 649                                 "Couldn't init rte_bbdev_op_data structs");
 650         }
 651
 652         return 0;
 653 }
 654
 655 static void
 656 free_buffers(struct active_device *ad, struct test_op_params *op_params)
 657 {
 658         unsigned int i, j;
 659
 660         rte_mempool_free(ad->ops_mempool);
 661         rte_mempool_free(ad->in_mbuf_pool);
 662         rte_mempool_free(ad->hard_out_mbuf_pool);
 663         rte_mempool_free(ad->soft_out_mbuf_pool);
 664
 665         for (i = 0; i < rte_lcore_count(); ++i) {
 666                 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) {
 667                         rte_free(op_params->q_bufs[j][i].inputs);
 668                         rte_free(op_params->q_bufs[j][i].hard_outputs);
 669                         rte_free(op_params->q_bufs[j][i].soft_outputs);
 670                 }
 671         }
 672 }
 673
 674 static void
 675 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
 676                 unsigned int start_idx,
 677                 struct rte_bbdev_op_data *inputs,
 678                 struct rte_bbdev_op_data *hard_outputs,
 679                 struct rte_bbdev_op_data *soft_outputs,
 680                 struct rte_bbdev_dec_op *ref_op)
 681 {
 682         unsigned int i;
 683         struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec;
 684
 685         for (i = 0; i < n; ++i) {
 686                 if (turbo_dec->code_block_mode == 0) {
 687                         ops[i]->turbo_dec.tb_params.ea =
 688                                         turbo_dec->tb_params.ea;
 689                         ops[i]->turbo_dec.tb_params.eb =
 690                                         turbo_dec->tb_params.eb;
 691                         ops[i]->turbo_dec.tb_params.k_pos =
 692                                         turbo_dec->tb_params.k_pos;
 693                         ops[i]->turbo_dec.tb_params.k_neg =
 694                                         turbo_dec->tb_params.k_neg;
 695                         ops[i]->turbo_dec.tb_params.c =
 696                                         turbo_dec->tb_params.c;
 697                         ops[i]->turbo_dec.tb_params.c_neg =
 698                                         turbo_dec->tb_params.c_neg;
 699                         ops[i]->turbo_dec.tb_params.cab =
 700                                         turbo_dec->tb_params.cab;
 701                 } else {
 702                         ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e;
 703                         ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k;
 704                 }
 705
 706                 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale;
 707                 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max;
 708                 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min;
 709                 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags;
 710                 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index;
 711                 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps;
 712                 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode;
 713
 714                 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i];
 715                 ops[i]->turbo_dec.input = inputs[start_idx + i];
 716                 if (soft_outputs != NULL)
 717                         ops[i]->turbo_dec.soft_output =
 718                                 soft_outputs[start_idx + i];
 719         }
 720 }
 721
 722 static void
 723 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n,
 724                 unsigned int start_idx,
 725                 struct rte_bbdev_op_data *inputs,
 726                 struct rte_bbdev_op_data *outputs,
 727                 struct rte_bbdev_enc_op *ref_op)
 728 {
 729         unsigned int i;
 730         struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc;
 731         for (i = 0; i < n; ++i) {
 732                 if (turbo_enc->code_block_mode == 0) {
 733                         ops[i]->turbo_enc.tb_params.ea =
 734                                         turbo_enc->tb_params.ea;
 735                         ops[i]->turbo_enc.tb_params.eb =
 736                                         turbo_enc->tb_params.eb;
 737                         ops[i]->turbo_enc.tb_params.k_pos =
 738                                         turbo_enc->tb_params.k_pos;
 739                         ops[i]->turbo_enc.tb_params.k_neg =
 740                                         turbo_enc->tb_params.k_neg;
 741                         ops[i]->turbo_enc.tb_params.c =
 742                                         turbo_enc->tb_params.c;
 743                         ops[i]->turbo_enc.tb_params.c_neg =
 744                                         turbo_enc->tb_params.c_neg;
 745                         ops[i]->turbo_enc.tb_params.cab =
 746                                         turbo_enc->tb_params.cab;
 747                         ops[i]->turbo_enc.tb_params.ncb_pos =
 748                                         turbo_enc->tb_params.ncb_pos;
 749                         ops[i]->turbo_enc.tb_params.ncb_neg =
 750                                         turbo_enc->tb_params.ncb_neg;
 751                         ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r;
 752                 } else {
 753                         ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e;
 754                         ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k;
 755                         ops[i]->turbo_enc.cb_params.ncb =
 756                                         turbo_enc->cb_params.ncb;
 757                 }
 758                 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index;
 759                 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags;
 760                 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode;
 761
 762                 ops[i]->turbo_enc.output = outputs[start_idx + i];
 763                 ops[i]->turbo_enc.input = inputs[start_idx + i];
 764         }
 765 }
 766
 767 static int
 768 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op,
 769                 unsigned int order_idx, const int expected_status)
 770 {
 771         TEST_ASSERT(op->status == expected_status,
 772                         "op_status (%d) != expected_status (%d)",
 773                         op->status, expected_status);
 774
 775         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
 776                         "Ordering error, expected %p, got %p",
 777                         (void *)(uintptr_t)order_idx, op->opaque_data);
 778
 779         return TEST_SUCCESS;
 780 }
 781
 782 static int
 783 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op,
 784                 unsigned int order_idx, const int expected_status)
 785 {
 786         TEST_ASSERT(op->status == expected_status,
 787                         "op_status (%d) != expected_status (%d)",
 788                         op->status, expected_status);
 789
 790         TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data,
 791                         "Ordering error, expected %p, got %p",
 792                         (void *)(uintptr_t)order_idx, op->opaque_data);
 793
 794         return TEST_SUCCESS;
 795 }
 796
 797 static inline int
 798 validate_op_chain(struct rte_bbdev_op_data *op,
 799                 struct op_data_entries *orig_op)
 800 {
 801         uint8_t i;
 802         struct rte_mbuf *m = op->data;
 803         uint8_t nb_dst_segments = orig_op->nb_segments;
 804
 805         TEST_ASSERT(nb_dst_segments == m->nb_segs,
 806                         "Number of segments differ in original (%u) and filled (%u) op",
 807                         nb_dst_segments, m->nb_segs);
 808
 809         for (i = 0; i < nb_dst_segments; ++i) {
 810                 /* Apply offset to the first mbuf segment */
 811                 uint16_t offset = (i == 0) ? op->offset : 0;
 812                 uint16_t data_len = m->data_len - offset;
 813
 814                 TEST_ASSERT(orig_op->segments[i].length == data_len,
 815                                 "Length of segment differ in original (%u) and filled (%u) op",
 816                                 orig_op->segments[i].length, data_len);
 817                 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr,
 818                                 rte_pktmbuf_mtod_offset(m, uint32_t *, offset),
 819                                 data_len,
 820                                 "Output buffers (CB=%u) are not equal", i);
 821                 m = m->next;
 822         }
 823
 824         return TEST_SUCCESS;
 825 }
 826
 827 static int
 828 validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs,
 829                 const uint16_t num_to_process)
 830 {
 831         int i;
 832
 833         struct op_data_entries *hard_data_orig =
 834                         &test_vector.entries[DATA_HARD_OUTPUT];
 835         struct op_data_entries *soft_data_orig =
 836                         &test_vector.entries[DATA_SOFT_OUTPUT];
 837
 838         for (i = 0; i < num_to_process; i++) {
 839                 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
 840                                 hard_data_orig),
 841                                 "Hard output buffers are not equal");
 842                 if (ref_op->turbo_dec.op_flags &
 843                                 RTE_BBDEV_TURBO_SOFT_OUTPUT)
 844                         TEST_ASSERT_SUCCESS(validate_op_chain(
 845                                         &bufs->soft_outputs[i],
 846                                         soft_data_orig),
 847                                         "Soft output buffers are not equal");
 848         }
 849
 850         return TEST_SUCCESS;
 851 }
 852
 853 static int
 854 validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process)
 855 {
 856         int i;
 857
 858         struct op_data_entries *hard_data_orig =
 859                         &test_vector.entries[DATA_HARD_OUTPUT];
 860
 861         for (i = 0; i < num_to_process; i++)
 862                 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i],
 863                                 hard_data_orig), "");
 864
 865         return TEST_SUCCESS;
 866 }
 867
 868 static int
 869 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
 870                 struct rte_bbdev_dec_op *ref_op, const int vector_mask)
 871 {
 872         unsigned int i;
 873         int ret;
 874         struct op_data_entries *hard_data_orig =
 875                         &test_vector.entries[DATA_HARD_OUTPUT];
 876         struct op_data_entries *soft_data_orig =
 877                         &test_vector.entries[DATA_SOFT_OUTPUT];
 878         struct rte_bbdev_op_turbo_dec *ops_td;
 879         struct rte_bbdev_op_data *hard_output;
 880         struct rte_bbdev_op_data *soft_output;
 881         struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec;
 882
 883         for (i = 0; i < n; ++i) {
 884                 ops_td = &ops[i]->turbo_dec;
 885                 hard_output = &ops_td->hard_output;
 886                 soft_output = &ops_td->soft_output;
 887
 888                 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
 889                         TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
 890                                         "Returned iter_count (%d) > expected iter_count (%d)",
 891                                         ops_td->iter_count, ref_td->iter_count);
 892                 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
 893                 TEST_ASSERT_SUCCESS(ret,
 894                                 "Checking status and ordering for decoder failed");
 895
 896                 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output,
 897                                 hard_data_orig),
 898                                 "Hard output buffers (CB=%u) are not equal",
 899                                 i);
 900
 901                 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)
 902                         TEST_ASSERT_SUCCESS(validate_op_chain(soft_output,
 903                                         soft_data_orig),
 904                                         "Soft output buffers (CB=%u) are not equal",
 905                                         i);
 906         }
 907
 908         return TEST_SUCCESS;
 909 }
 910
 911 static int
 912 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
 913                 struct rte_bbdev_enc_op *ref_op)
 914 {
 915         unsigned int i;
 916         int ret;
 917         struct op_data_entries *hard_data_orig =
 918                         &test_vector.entries[DATA_HARD_OUTPUT];
 919
 920         for (i = 0; i < n; ++i) {
 921                 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
 922                 TEST_ASSERT_SUCCESS(ret,
 923                                 "Checking status and ordering for encoder failed");
 924                 TEST_ASSERT_SUCCESS(validate_op_chain(
 925                                 &ops[i]->turbo_enc.output,
 926                                 hard_data_orig),
 927                                 "Output buffers (CB=%u) are not equal",
 928                                 i);
 929         }
 930
 931         return TEST_SUCCESS;
 932 }
 933
 934 static void
 935 create_reference_dec_op(struct rte_bbdev_dec_op *op)
 936 {
 937         unsigned int i;
 938         struct op_data_entries *entry;
 939
 940         op->turbo_dec = test_vector.turbo_dec;
 941         entry = &test_vector.entries[DATA_INPUT];
 942         for (i = 0; i < entry->nb_segments; ++i)
 943                 op->turbo_dec.input.length +=
 944                                 entry->segments[i].length;
 945 }
 946
 947 static void
 948 create_reference_enc_op(struct rte_bbdev_enc_op *op)
 949 {
 950         unsigned int i;
 951         struct op_data_entries *entry;
 952
 953         op->turbo_enc = test_vector.turbo_enc;
 954         entry = &test_vector.entries[DATA_INPUT];
 955         for (i = 0; i < entry->nb_segments; ++i)
 956                 op->turbo_enc.input.length +=
 957                                 entry->segments[i].length;
 958 }
 959
 960 static int
 961 init_test_op_params(struct test_op_params *op_params,
 962                 enum rte_bbdev_op_type op_type, const int expected_status,
 963                 const int vector_mask, struct rte_mempool *ops_mp,
 964                 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores)
 965 {
 966         int ret = 0;
 967         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
 968                 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp,
 969                                 &op_params->ref_dec_op, 1);
 970         else
 971                 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp,
 972                                 &op_params->ref_enc_op, 1);
 973
 974         TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed");
 975
 976         op_params->mp = ops_mp;
 977         op_params->burst_sz = burst_sz;
 978         op_params->num_to_process = num_to_process;
 979         op_params->num_lcores = num_lcores;
 980         op_params->vector_mask = vector_mask;
 981         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
 982                 op_params->ref_dec_op->status = expected_status;
 983         else if (op_type == RTE_BBDEV_OP_TURBO_ENC)
 984                 op_params->ref_enc_op->status = expected_status;
 985
 986         return 0;
 987 }
 988
 989 static int
 990 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id,
 991                 struct test_op_params *op_params)
 992 {
 993         int t_ret, f_ret, socket_id = SOCKET_ID_ANY;
 994         unsigned int i;
 995         struct active_device *ad;
 996         unsigned int burst_sz = get_burst_sz();
 997         enum rte_bbdev_op_type op_type = test_vector.op_type;
 998
 999         ad = &active_devs[dev_id];
1000
1001         /* Check if device supports op_type */
1002         if (!is_avail_op(ad, test_vector.op_type))
1003                 return TEST_SUCCESS;
1004
1005         struct rte_bbdev_info info;
1006         rte_bbdev_info_get(ad->dev_id, &info);
1007         socket_id = GET_SOCKET(info.socket_id);
1008
1009         if (op_type == RTE_BBDEV_OP_NONE)
1010                 op_type = RTE_BBDEV_OP_TURBO_ENC;
1011         f_ret = create_mempools(ad, socket_id, op_type,
1012                         get_num_ops());
1013         if (f_ret != TEST_SUCCESS) {
1014                 printf("Couldn't create mempools");
1015                 goto fail;
1016         }
1017
1018         f_ret = init_test_op_params(op_params, test_vector.op_type,
1019                         test_vector.expected_status,
1020                         test_vector.mask,
1021                         ad->ops_mempool,
1022                         burst_sz,
1023                         get_num_ops(),
1024                         get_num_lcores());
1025         if (f_ret != TEST_SUCCESS) {
1026                 printf("Couldn't init test op params");
1027                 goto fail;
1028         }
1029
1030         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1031                 create_reference_dec_op(op_params->ref_dec_op);
1032         else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1033                 create_reference_enc_op(op_params->ref_enc_op);
1034
1035         for (i = 0; i < ad->nb_queues; ++i) {
1036                 f_ret = fill_queue_buffers(op_params,
1037                                 ad->in_mbuf_pool,
1038                                 ad->hard_out_mbuf_pool,
1039                                 ad->soft_out_mbuf_pool,
1040                                 ad->queue_ids[i],
1041                                 info.drv.min_alignment,
1042                                 socket_id);
1043                 if (f_ret != TEST_SUCCESS) {
1044                         printf("Couldn't init queue buffers");
1045                         goto fail;
1046                 }
1047         }
1048
1049         /* Run test case function */
1050         t_ret = test_case_func(ad, op_params);
1051
1052         /* Free active device resources and return */
1053         free_buffers(ad, op_params);
1054         return t_ret;
1055
1056 fail:
1057         free_buffers(ad, op_params);
1058         return TEST_FAILED;
1059 }
1060
1061 /* Run given test function per active device per supported op type
1062  * per burst size.
1063  */
1064 static int
1065 run_test_case(test_case_function *test_case_func)
1066 {
1067         int ret = 0;
1068         uint8_t dev;
1069
1070         /* Alloc op_params */
1071         struct test_op_params *op_params = rte_zmalloc(NULL,
1072                         sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
1073         TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
1074                         RTE_ALIGN(sizeof(struct test_op_params),
1075                                 RTE_CACHE_LINE_SIZE));
1076
1077         /* For each device run test case function */
1078         for (dev = 0; dev < nb_active_devs; ++dev)
1079                 ret |= run_test_case_on_device(test_case_func, dev, op_params);
1080
1081         rte_free(op_params);
1082
1083         return ret;
1084 }
1085
1086 static void
1087 dequeue_event_callback(uint16_t dev_id,
1088                 enum rte_bbdev_event_type event, void *cb_arg,
1089                 void *ret_param)
1090 {
1091         int ret;
1092         uint16_t i;
1093         uint64_t total_time;
1094         uint16_t deq, burst_sz, num_to_process;
1095         uint16_t queue_id = INVALID_QUEUE_ID;
1096         struct rte_bbdev_dec_op *dec_ops[MAX_BURST];
1097         struct rte_bbdev_enc_op *enc_ops[MAX_BURST];
1098         struct test_buffers *bufs;
1099         struct rte_bbdev_info info;
1100
1101         /* Input length in bytes, million operations per second,
1102          * million bits per second.
1103          */
1104         double in_len;
1105
1106         struct thread_params *tp = cb_arg;
1107
1108         RTE_SET_USED(ret_param);
1109         queue_id = tp->queue_id;
1110
1111         /* Find matching thread params using queue_id */
1112         for (i = 0; i < MAX_QUEUES; ++i, ++tp)
1113                 if (tp->queue_id == queue_id)
1114                         break;
1115
1116         if (i == MAX_QUEUES) {
1117                 printf("%s: Queue_id from interrupt details was not found!\n",
1118                                 __func__);
1119                 return;
1120         }
1121
1122         if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) {
1123                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1124                 printf(
1125                         "Dequeue interrupt handler called for incorrect event!\n");
1126                 return;
1127         }
1128
1129         burst_sz = tp->op_params->burst_sz;
1130         num_to_process = tp->op_params->num_to_process;
1131
1132         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1133                 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops,
1134                                 burst_sz);
1135         else
1136                 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops,
1137                                 burst_sz);
1138
1139         if (deq < burst_sz) {
1140                 printf(
1141                         "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n",
1142                         burst_sz, deq);
1143                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1144                 return;
1145         }
1146
1147         if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) {
1148                 rte_atomic16_add(&tp->nb_dequeued, deq);
1149                 return;
1150         }
1151
1152         total_time = rte_rdtsc_precise() - tp->start_time;
1153
1154         rte_bbdev_info_get(dev_id, &info);
1155
1156         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1157
1158         ret = TEST_SUCCESS;
1159         if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1160                 ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs,
1161                                 num_to_process);
1162         else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC)
1163                 ret = validate_enc_buffers(bufs, num_to_process);
1164
1165         if (ret) {
1166                 printf("Buffers validation failed\n");
1167                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1168         }
1169
1170         switch (test_vector.op_type) {
1171         case RTE_BBDEV_OP_TURBO_DEC:
1172                 in_len = tp->op_params->ref_dec_op->turbo_dec.input.length;
1173                 break;
1174         case RTE_BBDEV_OP_TURBO_ENC:
1175                 in_len = tp->op_params->ref_enc_op->turbo_enc.input.length;
1176                 break;
1177         case RTE_BBDEV_OP_NONE:
1178                 in_len = 0.0;
1179                 break;
1180         default:
1181                 printf("Unknown op type: %d\n", test_vector.op_type);
1182                 rte_atomic16_set(&tp->processing_status, TEST_FAILED);
1183                 return;
1184         }
1185
1186         tp->mops = ((double)num_to_process / 1000000.0) /
1187                         ((double)total_time / (double)rte_get_tsc_hz());
1188         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1189                         ((double)total_time / (double)rte_get_tsc_hz());
1190
1191         rte_atomic16_add(&tp->nb_dequeued, deq);
1192 }
1193
1194 static int
1195 throughput_intr_lcore_dec(void *arg)
1196 {
1197         struct thread_params *tp = arg;
1198         unsigned int enqueued;
1199         struct rte_bbdev_dec_op *ops[MAX_BURST];
1200         const uint16_t queue_id = tp->queue_id;
1201         const uint16_t burst_sz = tp->op_params->burst_sz;
1202         const uint16_t num_to_process = tp->op_params->num_to_process;
1203         struct test_buffers *bufs = NULL;
1204         unsigned int allocs_failed = 0;
1205         struct rte_bbdev_info info;
1206         int ret;
1207
1208         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1209                         "BURST_SIZE should be <= %u", MAX_BURST);
1210
1211         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1212                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1213                         tp->dev_id, queue_id);
1214
1215         rte_bbdev_info_get(tp->dev_id, &info);
1216         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1217
1218         rte_atomic16_clear(&tp->processing_status);
1219         rte_atomic16_clear(&tp->nb_dequeued);
1220
1221         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1222                 rte_pause();
1223
1224         tp->start_time = rte_rdtsc_precise();
1225         for (enqueued = 0; enqueued < num_to_process;) {
1226
1227                 uint16_t num_to_enq = burst_sz;
1228
1229                 if (unlikely(num_to_process - enqueued < num_to_enq))
1230                         num_to_enq = num_to_process - enqueued;
1231
1232                 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops,
1233                                 num_to_enq);
1234                 if (ret != 0) {
1235                         allocs_failed++;
1236                         continue;
1237                 }
1238
1239                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1240                         copy_reference_dec_op(ops, num_to_enq, enqueued,
1241                                         bufs->inputs,
1242                                         bufs->hard_outputs,
1243                                         bufs->soft_outputs,
1244                                         tp->op_params->ref_dec_op);
1245
1246                 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops,
1247                                 num_to_enq);
1248
1249                 rte_bbdev_dec_op_free_bulk(ops, num_to_enq);
1250         }
1251
1252         if (allocs_failed > 0)
1253                 printf("WARNING: op allocations failed: %u times\n",
1254                                 allocs_failed);
1255
1256         return TEST_SUCCESS;
1257 }
1258
1259 static int
1260 throughput_intr_lcore_enc(void *arg)
1261 {
1262         struct thread_params *tp = arg;
1263         unsigned int enqueued;
1264         struct rte_bbdev_enc_op *ops[MAX_BURST];
1265         const uint16_t queue_id = tp->queue_id;
1266         const uint16_t burst_sz = tp->op_params->burst_sz;
1267         const uint16_t num_to_process = tp->op_params->num_to_process;
1268         struct test_buffers *bufs = NULL;
1269         unsigned int allocs_failed = 0;
1270         struct rte_bbdev_info info;
1271         int ret;
1272
1273         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1274                         "BURST_SIZE should be <= %u", MAX_BURST);
1275
1276         TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id),
1277                         "Failed to enable interrupts for dev: %u, queue_id: %u",
1278                         tp->dev_id, queue_id);
1279
1280         rte_bbdev_info_get(tp->dev_id, &info);
1281         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1282
1283         rte_atomic16_clear(&tp->processing_status);
1284         rte_atomic16_clear(&tp->nb_dequeued);
1285
1286         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1287                 rte_pause();
1288
1289         tp->start_time = rte_rdtsc_precise();
1290         for (enqueued = 0; enqueued < num_to_process;) {
1291
1292                 uint16_t num_to_enq = burst_sz;
1293
1294                 if (unlikely(num_to_process - enqueued < num_to_enq))
1295                         num_to_enq = num_to_process - enqueued;
1296
1297                 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops,
1298                                 num_to_enq);
1299                 if (ret != 0) {
1300                         allocs_failed++;
1301                         continue;
1302                 }
1303
1304                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1305                         copy_reference_enc_op(ops, num_to_enq, enqueued,
1306                                         bufs->inputs,
1307                                         bufs->hard_outputs,
1308                                         tp->op_params->ref_enc_op);
1309
1310                 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops,
1311                                 num_to_enq);
1312
1313                 rte_bbdev_enc_op_free_bulk(ops, num_to_enq);
1314         }
1315
1316         if (allocs_failed > 0)
1317                 printf("WARNING: op allocations failed: %u times\n",
1318                                 allocs_failed);
1319
1320         return TEST_SUCCESS;
1321 }
1322
1323 static int
1324 throughput_pmd_lcore_dec(void *arg)
1325 {
1326         struct thread_params *tp = arg;
1327         unsigned int enqueued, dequeued;
1328         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1329         uint64_t total_time, start_time;
1330         const uint16_t queue_id = tp->queue_id;
1331         const uint16_t burst_sz = tp->op_params->burst_sz;
1332         const uint16_t num_to_process = tp->op_params->num_to_process;
1333         struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op;
1334         struct test_buffers *bufs = NULL;
1335         unsigned int allocs_failed = 0;
1336         int ret;
1337         struct rte_bbdev_info info;
1338
1339         /* Input length in bytes, million operations per second, million bits
1340          * per second.
1341          */
1342         double in_len;
1343
1344         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1345                         "BURST_SIZE should be <= %u", MAX_BURST);
1346
1347         rte_bbdev_info_get(tp->dev_id, &info);
1348         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1349
1350         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1351                 rte_pause();
1352
1353         start_time = rte_rdtsc_precise();
1354         for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1355                 uint16_t deq;
1356
1357                 if (likely(enqueued < num_to_process)) {
1358
1359                         uint16_t num_to_enq = burst_sz;
1360
1361                         if (unlikely(num_to_process - enqueued < num_to_enq))
1362                                 num_to_enq = num_to_process - enqueued;
1363
1364                         ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp,
1365                                         ops_enq, num_to_enq);
1366                         if (ret != 0) {
1367                                 allocs_failed++;
1368                                 goto do_dequeue;
1369                         }
1370
1371                         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1372                                 copy_reference_dec_op(ops_enq, num_to_enq,
1373                                                 enqueued,
1374                                                 bufs->inputs,
1375                                                 bufs->hard_outputs,
1376                                                 bufs->soft_outputs,
1377                                                 ref_op);
1378
1379                         enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id,
1380                                         queue_id, ops_enq, num_to_enq);
1381                 }
1382 do_dequeue:
1383                 deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq,
1384                                 burst_sz);
1385                 dequeued += deq;
1386                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1387         }
1388         total_time = rte_rdtsc_precise() - start_time;
1389
1390         if (allocs_failed > 0)
1391                 printf("WARNING: op allocations failed: %u times\n",
1392                                 allocs_failed);
1393
1394         TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1395                         enqueued, dequeued);
1396
1397         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1398                 ret = validate_dec_buffers(ref_op, bufs, num_to_process);
1399                 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1400         }
1401
1402         in_len = ref_op->turbo_dec.input.length;
1403         tp->mops = ((double)num_to_process / 1000000.0) /
1404                         ((double)total_time / (double)rte_get_tsc_hz());
1405         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1406                         ((double)total_time / (double)rte_get_tsc_hz());
1407
1408         return TEST_SUCCESS;
1409 }
1410
1411 static int
1412 throughput_pmd_lcore_enc(void *arg)
1413 {
1414         struct thread_params *tp = arg;
1415         unsigned int enqueued, dequeued;
1416         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1417         uint64_t total_time, start_time;
1418         const uint16_t queue_id = tp->queue_id;
1419         const uint16_t burst_sz = tp->op_params->burst_sz;
1420         const uint16_t num_to_process = tp->op_params->num_to_process;
1421         struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op;
1422         struct test_buffers *bufs = NULL;
1423         unsigned int allocs_failed = 0;
1424         int ret;
1425         struct rte_bbdev_info info;
1426
1427         /* Input length in bytes, million operations per second, million bits
1428          * per second.
1429          */
1430         double in_len;
1431
1432         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1433                         "BURST_SIZE should be <= %u", MAX_BURST);
1434
1435         rte_bbdev_info_get(tp->dev_id, &info);
1436         bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1437
1438         while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT)
1439                 rte_pause();
1440
1441         start_time = rte_rdtsc_precise();
1442         for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) {
1443                 uint16_t deq;
1444
1445                 if (likely(enqueued < num_to_process)) {
1446
1447                         uint16_t num_to_enq = burst_sz;
1448
1449                         if (unlikely(num_to_process - enqueued < num_to_enq))
1450                                 num_to_enq = num_to_process - enqueued;
1451
1452                         ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp,
1453                                         ops_enq, num_to_enq);
1454                         if (ret != 0) {
1455                                 allocs_failed++;
1456                                 goto do_dequeue;
1457                         }
1458
1459                         if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1460                                 copy_reference_enc_op(ops_enq, num_to_enq,
1461                                                 enqueued,
1462                                                 bufs->inputs,
1463                                                 bufs->hard_outputs,
1464                                                 ref_op);
1465
1466                         enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id,
1467                                         queue_id, ops_enq, num_to_enq);
1468                 }
1469 do_dequeue:
1470                 deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq,
1471                                 burst_sz);
1472                 dequeued += deq;
1473                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1474         }
1475         total_time = rte_rdtsc_precise() - start_time;
1476
1477         if (allocs_failed > 0)
1478                 printf("WARNING: op allocations failed: %u times\n",
1479                                 allocs_failed);
1480
1481         TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)",
1482                         enqueued, dequeued);
1483
1484         if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1485                 ret = validate_enc_buffers(bufs, num_to_process);
1486                 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed");
1487         }
1488
1489         in_len = ref_op->turbo_enc.input.length;
1490
1491         tp->mops = ((double)num_to_process / 1000000.0) /
1492                         ((double)total_time / (double)rte_get_tsc_hz());
1493         tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) /
1494                         ((double)total_time / (double)rte_get_tsc_hz());
1495
1496         return TEST_SUCCESS;
1497 }
1498 static void
1499 print_throughput(struct thread_params *t_params, unsigned int used_cores)
1500 {
1501         unsigned int lcore_id, iter = 0;
1502         double total_mops = 0, total_mbps = 0;
1503
1504         RTE_LCORE_FOREACH(lcore_id) {
1505                 if (iter++ >= used_cores)
1506                         break;
1507                 printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n",
1508                 lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps);
1509                 total_mops += t_params[lcore_id].mops;
1510                 total_mbps += t_params[lcore_id].mbps;
1511         }
1512         printf(
1513                 "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n",
1514                 used_cores, total_mops, total_mbps);
1515 }
1516
1517 /*
1518  * Test function that determines how long an enqueue + dequeue of a burst
1519  * takes on available lcores.
1520  */
1521 static int
1522 throughput_test(struct active_device *ad,
1523                 struct test_op_params *op_params)
1524 {
1525         int ret;
1526         unsigned int lcore_id, used_cores = 0;
1527         struct thread_params t_params[MAX_QUEUES];
1528         struct rte_bbdev_info info;
1529         lcore_function_t *throughput_function;
1530         struct thread_params *tp;
1531         uint16_t num_lcores;
1532         const char *op_type_str;
1533
1534         rte_bbdev_info_get(ad->dev_id, &info);
1535
1536         op_type_str = rte_bbdev_op_type_str(test_vector.op_type);
1537         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u",
1538                         test_vector.op_type);
1539
1540         printf(
1541                 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n",
1542                         info.dev_name, ad->nb_queues, op_params->burst_sz,
1543                         op_params->num_to_process, op_params->num_lcores,
1544                         op_type_str,
1545                         intr_enabled ? "Interrupt mode" : "PMD mode",
1546                         (double)rte_get_tsc_hz() / 1000000000.0);
1547
1548         /* Set number of lcores */
1549         num_lcores = (ad->nb_queues < (op_params->num_lcores))
1550                         ? ad->nb_queues
1551                         : op_params->num_lcores;
1552
1553         if (intr_enabled) {
1554                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1555                         throughput_function = throughput_intr_lcore_dec;
1556                 else
1557                         throughput_function = throughput_intr_lcore_enc;
1558
1559                 /* Dequeue interrupt callback registration */
1560                 ret = rte_bbdev_callback_register(ad->dev_id,
1561                                 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback,
1562                                 &t_params);
1563                 if (ret < 0)
1564                         return ret;
1565         } else {
1566                 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC)
1567                         throughput_function = throughput_pmd_lcore_dec;
1568                 else
1569                         throughput_function = throughput_pmd_lcore_enc;
1570         }
1571
1572         rte_atomic16_set(&op_params->sync, SYNC_WAIT);
1573
1574         t_params[rte_lcore_id()].dev_id = ad->dev_id;
1575         t_params[rte_lcore_id()].op_params = op_params;
1576         t_params[rte_lcore_id()].queue_id =
1577                         ad->queue_ids[used_cores++];
1578
1579         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1580                 if (used_cores >= num_lcores)
1581                         break;
1582
1583                 t_params[lcore_id].dev_id = ad->dev_id;
1584                 t_params[lcore_id].op_params = op_params;
1585                 t_params[lcore_id].queue_id = ad->queue_ids[used_cores++];
1586
1587                 rte_eal_remote_launch(throughput_function, &t_params[lcore_id],
1588                                 lcore_id);
1589         }
1590
1591         rte_atomic16_set(&op_params->sync, SYNC_START);
1592         ret = throughput_function(&t_params[rte_lcore_id()]);
1593
1594         /* Master core is always used */
1595         used_cores = 1;
1596         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1597                 if (used_cores++ >= num_lcores)
1598                         break;
1599
1600                 ret |= rte_eal_wait_lcore(lcore_id);
1601         }
1602
1603         /* Return if test failed */
1604         if (ret)
1605                 return ret;
1606
1607         /* Print throughput if interrupts are disabled and test passed */
1608         if (!intr_enabled) {
1609                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1610                         print_throughput(t_params, num_lcores);
1611                 return ret;
1612         }
1613
1614         /* In interrupt TC we need to wait for the interrupt callback to deqeue
1615          * all pending operations. Skip waiting for queues which reported an
1616          * error using processing_status variable.
1617          * Wait for master lcore operations.
1618          */
1619         tp = &t_params[rte_lcore_id()];
1620         while ((rte_atomic16_read(&tp->nb_dequeued) <
1621                         op_params->num_to_process) &&
1622                         (rte_atomic16_read(&tp->processing_status) !=
1623                         TEST_FAILED))
1624                 rte_pause();
1625
1626         ret |= rte_atomic16_read(&tp->processing_status);
1627
1628         /* Wait for slave lcores operations */
1629         used_cores = 1;
1630         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1631                 tp = &t_params[lcore_id];
1632                 if (used_cores++ >= num_lcores)
1633                         break;
1634
1635                 while ((rte_atomic16_read(&tp->nb_dequeued) <
1636                                 op_params->num_to_process) &&
1637                                 (rte_atomic16_read(&tp->processing_status) !=
1638                                 TEST_FAILED))
1639                         rte_pause();
1640
1641                 ret |= rte_atomic16_read(&tp->processing_status);
1642         }
1643
1644         /* Print throughput if test passed */
1645         if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE)
1646                 print_throughput(t_params, num_lcores);
1647
1648         return ret;
1649 }
1650
1651 static int
1652 operation_latency_test_dec(struct rte_mempool *mempool,
1653                 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op,
1654                 int vector_mask, uint16_t dev_id, uint16_t queue_id,
1655                 const uint16_t num_to_process, uint16_t burst_sz,
1656                 uint64_t *total_time)
1657 {
1658         int ret = TEST_SUCCESS;
1659         uint16_t i, j, dequeued;
1660         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1661         uint64_t start_time = 0;
1662
1663         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1664                 uint16_t enq = 0, deq = 0;
1665                 bool first_time = true;
1666
1667                 if (unlikely(num_to_process - dequeued < burst_sz))
1668                         burst_sz = num_to_process - dequeued;
1669
1670                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1671                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1672                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1673                                         bufs->inputs,
1674                                         bufs->hard_outputs,
1675                                         bufs->soft_outputs,
1676                                         ref_op);
1677
1678                 /* Set counter to validate the ordering */
1679                 for (j = 0; j < burst_sz; ++j)
1680                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1681
1682                 start_time = rte_rdtsc_precise();
1683
1684                 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq],
1685                                 burst_sz);
1686                 TEST_ASSERT(enq == burst_sz,
1687                                 "Error enqueueing burst, expected %u, got %u",
1688                                 burst_sz, enq);
1689
1690                 /* Dequeue */
1691                 do {
1692                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1693                                         &ops_deq[deq], burst_sz - deq);
1694                         if (likely(first_time && (deq > 0))) {
1695                                 *total_time += rte_rdtsc_precise() - start_time;
1696                                 first_time = false;
1697                         }
1698                 } while (unlikely(burst_sz != deq));
1699
1700                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1701                         ret = validate_dec_op(ops_deq, burst_sz, ref_op,
1702                                         vector_mask);
1703                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1704                 }
1705
1706                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1707                 dequeued += deq;
1708         }
1709
1710         return i;
1711 }
1712
1713 static int
1714 operation_latency_test_enc(struct rte_mempool *mempool,
1715                 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op,
1716                 uint16_t dev_id, uint16_t queue_id,
1717                 const uint16_t num_to_process, uint16_t burst_sz,
1718                 uint64_t *total_time)
1719 {
1720         int ret = TEST_SUCCESS;
1721         uint16_t i, j, dequeued;
1722         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1723         uint64_t start_time = 0;
1724
1725         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1726                 uint16_t enq = 0, deq = 0;
1727                 bool first_time = true;
1728
1729                 if (unlikely(num_to_process - dequeued < burst_sz))
1730                         burst_sz = num_to_process - dequeued;
1731
1732                 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1733                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1734                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1735                                         bufs->inputs,
1736                                         bufs->hard_outputs,
1737                                         ref_op);
1738
1739                 /* Set counter to validate the ordering */
1740                 for (j = 0; j < burst_sz; ++j)
1741                         ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
1742
1743                 start_time = rte_rdtsc_precise();
1744
1745                 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq],
1746                                 burst_sz);
1747                 TEST_ASSERT(enq == burst_sz,
1748                                 "Error enqueueing burst, expected %u, got %u",
1749                                 burst_sz, enq);
1750
1751                 /* Dequeue */
1752                 do {
1753                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1754                                         &ops_deq[deq], burst_sz - deq);
1755                         if (likely(first_time && (deq > 0))) {
1756                                 *total_time += rte_rdtsc_precise() - start_time;
1757                                 first_time = false;
1758                         }
1759                 } while (unlikely(burst_sz != deq));
1760
1761                 if (test_vector.op_type != RTE_BBDEV_OP_NONE) {
1762                         ret = validate_enc_op(ops_deq, burst_sz, ref_op);
1763                         TEST_ASSERT_SUCCESS(ret, "Validation failed!");
1764                 }
1765
1766                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1767                 dequeued += deq;
1768         }
1769
1770         return i;
1771 }
1772
1773 static int
1774 operation_latency_test(struct active_device *ad,
1775                 struct test_op_params *op_params)
1776 {
1777         int iter;
1778         uint16_t burst_sz = op_params->burst_sz;
1779         const uint16_t num_to_process = op_params->num_to_process;
1780         const enum rte_bbdev_op_type op_type = test_vector.op_type;
1781         const uint16_t queue_id = ad->queue_ids[0];
1782         struct test_buffers *bufs = NULL;
1783         struct rte_bbdev_info info;
1784         uint64_t total_time = 0;
1785         const char *op_type_str;
1786
1787         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1788                         "BURST_SIZE should be <= %u", MAX_BURST);
1789
1790         rte_bbdev_info_get(ad->dev_id, &info);
1791         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1792
1793         op_type_str = rte_bbdev_op_type_str(op_type);
1794         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
1795
1796         printf(
1797                 "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
1798                         info.dev_name, burst_sz, num_to_process, op_type_str);
1799
1800         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1801                 iter = operation_latency_test_dec(op_params->mp, bufs,
1802                                 op_params->ref_dec_op, op_params->vector_mask,
1803                                 ad->dev_id, queue_id, num_to_process,
1804                                 burst_sz, &total_time);
1805         else
1806                 iter = operation_latency_test_enc(op_params->mp, bufs,
1807                                 op_params->ref_enc_op, ad->dev_id, queue_id,
1808                                 num_to_process, burst_sz, &total_time);
1809
1810         if (iter <= 0)
1811                 return TEST_FAILED;
1812
1813         printf("\toperation avg. latency: %lg cycles, %lg us\n",
1814                         (double)total_time / (double)iter,
1815                         (double)(total_time * 1000000) / (double)iter /
1816                         (double)rte_get_tsc_hz());
1817
1818         return TEST_SUCCESS;
1819 }
1820
1821 static int
1822 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs,
1823                 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id,
1824                 uint16_t queue_id, const uint16_t num_to_process,
1825                 uint16_t burst_sz, uint64_t *enq_total_time,
1826                 uint64_t *deq_total_time)
1827 {
1828         int i, dequeued;
1829         struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1830         uint64_t enq_start_time, deq_start_time;
1831
1832         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1833                 uint16_t enq = 0, deq = 0;
1834
1835                 if (unlikely(num_to_process - dequeued < burst_sz))
1836                         burst_sz = num_to_process - dequeued;
1837
1838                 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz);
1839                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1840                         copy_reference_dec_op(ops_enq, burst_sz, dequeued,
1841                                         bufs->inputs,
1842                                         bufs->hard_outputs,
1843                                         bufs->soft_outputs,
1844                                         ref_op);
1845
1846                 /* Start time measurment for enqueue function offload latency */
1847                 enq_start_time = rte_rdtsc();
1848                 do {
1849                         enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id,
1850                                         &ops_enq[enq], burst_sz - enq);
1851                 } while (unlikely(burst_sz != enq));
1852                 *enq_total_time += rte_rdtsc() - enq_start_time;
1853
1854                 /* ensure enqueue has been completed */
1855                 rte_delay_ms(10);
1856
1857                 /* Start time measurment for dequeue function offload latency */
1858                 deq_start_time = rte_rdtsc();
1859                 do {
1860                         deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id,
1861                                         &ops_deq[deq], burst_sz - deq);
1862                 } while (unlikely(burst_sz != deq));
1863                 *deq_total_time += rte_rdtsc() - deq_start_time;
1864
1865                 rte_bbdev_dec_op_free_bulk(ops_enq, deq);
1866                 dequeued += deq;
1867         }
1868
1869         return i;
1870 }
1871
1872 static int
1873 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs,
1874                 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id,
1875                 uint16_t queue_id, const uint16_t num_to_process,
1876                 uint16_t burst_sz, uint64_t *enq_total_time,
1877                 uint64_t *deq_total_time)
1878 {
1879         int i, dequeued;
1880         struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST];
1881         uint64_t enq_start_time, deq_start_time;
1882
1883         for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) {
1884                 uint16_t enq = 0, deq = 0;
1885
1886                 if (unlikely(num_to_process - dequeued < burst_sz))
1887                         burst_sz = num_to_process - dequeued;
1888
1889                 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz);
1890                 if (test_vector.op_type != RTE_BBDEV_OP_NONE)
1891                         copy_reference_enc_op(ops_enq, burst_sz, dequeued,
1892                                         bufs->inputs,
1893                                         bufs->hard_outputs,
1894                                         ref_op);
1895
1896                 /* Start time measurment for enqueue function offload latency */
1897                 enq_start_time = rte_rdtsc();
1898                 do {
1899                         enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id,
1900                                         &ops_enq[enq], burst_sz - enq);
1901                 } while (unlikely(burst_sz != enq));
1902                 *enq_total_time += rte_rdtsc() - enq_start_time;
1903
1904                 /* ensure enqueue has been completed */
1905                 rte_delay_ms(10);
1906
1907                 /* Start time measurment for dequeue function offload latency */
1908                 deq_start_time = rte_rdtsc();
1909                 do {
1910                         deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id,
1911                                         &ops_deq[deq], burst_sz - deq);
1912                 } while (unlikely(burst_sz != deq));
1913                 *deq_total_time += rte_rdtsc() - deq_start_time;
1914
1915                 rte_bbdev_enc_op_free_bulk(ops_enq, deq);
1916                 dequeued += deq;
1917         }
1918
1919         return i;
1920 }
1921
1922 static int
1923 offload_latency_test(struct active_device *ad,
1924                 struct test_op_params *op_params)
1925 {
1926         int iter;
1927         uint64_t enq_total_time = 0, deq_total_time = 0;
1928         uint16_t burst_sz = op_params->burst_sz;
1929         const uint16_t num_to_process = op_params->num_to_process;
1930         const enum rte_bbdev_op_type op_type = test_vector.op_type;
1931         const uint16_t queue_id = ad->queue_ids[0];
1932         struct test_buffers *bufs = NULL;
1933         struct rte_bbdev_info info;
1934         const char *op_type_str;
1935
1936         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
1937                         "BURST_SIZE should be <= %u", MAX_BURST);
1938
1939         rte_bbdev_info_get(ad->dev_id, &info);
1940         bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id];
1941
1942         op_type_str = rte_bbdev_op_type_str(op_type);
1943         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
1944
1945         printf(
1946                 "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
1947                         info.dev_name, burst_sz, num_to_process, op_type_str);
1948
1949         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
1950                 iter = offload_latency_test_dec(op_params->mp, bufs,
1951                                 op_params->ref_dec_op, ad->dev_id, queue_id,
1952                                 num_to_process, burst_sz, &enq_total_time,
1953                                 &deq_total_time);
1954         else
1955                 iter = offload_latency_test_enc(op_params->mp, bufs,
1956                                 op_params->ref_enc_op, ad->dev_id, queue_id,
1957                                 num_to_process, burst_sz, &enq_total_time,
1958                                 &deq_total_time);
1959
1960         if (iter <= 0)
1961                 return TEST_FAILED;
1962
1963         printf("\tenq offload avg. latency: %lg cycles, %lg us\n",
1964                         (double)enq_total_time / (double)iter,
1965                         (double)(enq_total_time * 1000000) / (double)iter /
1966                         (double)rte_get_tsc_hz());
1967
1968         printf("\tdeq offload avg. latency: %lg cycles, %lg us\n",
1969                         (double)deq_total_time / (double)iter,
1970                         (double)(deq_total_time * 1000000) / (double)iter /
1971                         (double)rte_get_tsc_hz());
1972
1973         return TEST_SUCCESS;
1974 }
1975
1976 static int
1977 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id,
1978                 const uint16_t num_to_process, uint16_t burst_sz,
1979                 uint64_t *deq_total_time)
1980 {
1981         int i, deq_total;
1982         struct rte_bbdev_dec_op *ops[MAX_BURST];
1983         uint64_t deq_start_time;
1984
1985         /* Test deq offload latency from an empty queue */
1986         deq_start_time = rte_rdtsc_precise();
1987         for (i = 0, deq_total = 0; deq_total < num_to_process;
1988                         ++i, deq_total += burst_sz) {
1989                 if (unlikely(num_to_process - deq_total < burst_sz))
1990                         burst_sz = num_to_process - deq_total;
1991                 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz);
1992         }
1993         *deq_total_time = rte_rdtsc_precise() - deq_start_time;
1994
1995         return i;
1996 }
1997
1998 static int
1999 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id,
2000                 const uint16_t num_to_process, uint16_t burst_sz,
2001                 uint64_t *deq_total_time)
2002 {
2003         int i, deq_total;
2004         struct rte_bbdev_enc_op *ops[MAX_BURST];
2005         uint64_t deq_start_time;
2006
2007         /* Test deq offload latency from an empty queue */
2008         deq_start_time = rte_rdtsc_precise();
2009         for (i = 0, deq_total = 0; deq_total < num_to_process;
2010                         ++i, deq_total += burst_sz) {
2011                 if (unlikely(num_to_process - deq_total < burst_sz))
2012                         burst_sz = num_to_process - deq_total;
2013                 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz);
2014         }
2015         *deq_total_time = rte_rdtsc_precise() - deq_start_time;
2016
2017         return i;
2018 }
2019
2020 static int
2021 offload_latency_empty_q_test(struct active_device *ad,
2022                 struct test_op_params *op_params)
2023 {
2024         int iter;
2025         uint64_t deq_total_time = 0;
2026         uint16_t burst_sz = op_params->burst_sz;
2027         const uint16_t num_to_process = op_params->num_to_process;
2028         const enum rte_bbdev_op_type op_type = test_vector.op_type;
2029         const uint16_t queue_id = ad->queue_ids[0];
2030         struct rte_bbdev_info info;
2031         const char *op_type_str;
2032
2033         TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST),
2034                         "BURST_SIZE should be <= %u", MAX_BURST);
2035
2036         rte_bbdev_info_get(ad->dev_id, &info);
2037
2038         op_type_str = rte_bbdev_op_type_str(op_type);
2039         TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type);
2040
2041         printf(
2042                 "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n",
2043                         info.dev_name, burst_sz, num_to_process, op_type_str);
2044
2045         if (op_type == RTE_BBDEV_OP_TURBO_DEC)
2046                 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id,
2047                                 num_to_process, burst_sz, &deq_total_time);
2048         else
2049                 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id,
2050                                 num_to_process, burst_sz, &deq_total_time);
2051
2052         if (iter <= 0)
2053                 return TEST_FAILED;
2054
2055         printf("\tempty deq offload avg. latency: %lg cycles, %lg us\n",
2056                         (double)deq_total_time / (double)iter,
2057                         (double)(deq_total_time * 1000000) / (double)iter /
2058                         (double)rte_get_tsc_hz());
2059
2060         return TEST_SUCCESS;
2061 }
2062
2063 static int
2064 throughput_tc(void)
2065 {
2066         return run_test_case(throughput_test);
2067 }
2068
2069 static int
2070 offload_latency_tc(void)
2071 {
2072         return run_test_case(offload_latency_test);
2073 }
2074
2075 static int
2076 offload_latency_empty_q_tc(void)
2077 {
2078         return run_test_case(offload_latency_empty_q_test);
2079 }
2080
2081 static int
2082 operation_latency_tc(void)
2083 {
2084         return run_test_case(operation_latency_test);
2085 }
2086
2087 static int
2088 interrupt_tc(void)
2089 {
2090         return run_test_case(throughput_test);
2091 }
2092
2093 static struct unit_test_suite bbdev_throughput_testsuite = {
2094         .suite_name = "BBdev Throughput Tests",
2095         .setup = testsuite_setup,
2096         .teardown = testsuite_teardown,
2097         .unit_test_cases = {
2098                 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc),
2099                 TEST_CASES_END() /**< NULL terminate unit test array */
2100         }
2101 };
2102
2103 static struct unit_test_suite bbdev_validation_testsuite = {
2104         .suite_name = "BBdev Validation Tests",
2105         .setup = testsuite_setup,
2106         .teardown = testsuite_teardown,
2107         .unit_test_cases = {
2108                 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc),
2109                 TEST_CASES_END() /**< NULL terminate unit test array */
2110         }
2111 };
2112
2113 static struct unit_test_suite bbdev_latency_testsuite = {
2114         .suite_name = "BBdev Latency Tests",
2115         .setup = testsuite_setup,
2116         .teardown = testsuite_teardown,
2117         .unit_test_cases = {
2118                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_tc),
2119                 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc),
2120                 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc),
2121                 TEST_CASES_END() /**< NULL terminate unit test array */
2122         }
2123 };
2124
2125 static struct unit_test_suite bbdev_interrupt_testsuite = {
2126         .suite_name = "BBdev Interrupt Tests",
2127         .setup = interrupt_testsuite_setup,
2128         .teardown = testsuite_teardown,
2129         .unit_test_cases = {
2130                 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc),
2131                 TEST_CASES_END() /**< NULL terminate unit test array */
2132         }
2133 };
2134
2135 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite);
2136 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite);
2137 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite);
2138 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite);