test/test/test_eventdev_octeontx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Cavium, Inc. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *       * Redistributions of source code must retain the above copyright
  11  *         notice, this list of conditions and the following disclaimer.
  12  *       * Redistributions in binary form must reproduce the above copyright
  13  *         notice, this list of conditions and the following disclaimer in
  14  *         the documentation and/or other materials provided with the
  15  *         distribution.
  16  *       * Neither the name of Cavium, Inc nor the names of its
  17  *         contributors may be used to endorse or promote products derived
  18  *         from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_atomic.h>
  34 #include <rte_common.h>
  35 #include <rte_cycles.h>
  36 #include <rte_debug.h>
  37 #include <rte_eal.h>
  38 #include <rte_ethdev.h>
  39 #include <rte_eventdev.h>
  40 #include <rte_hexdump.h>
  41 #include <rte_mbuf.h>
  42 #include <rte_malloc.h>
  43 #include <rte_memcpy.h>
  44 #include <rte_launch.h>
  45 #include <rte_lcore.h>
  46 #include <rte_per_lcore.h>
  47 #include <rte_random.h>
  48 #include <rte_bus_vdev.h>
  49
  50 #include "test.h"
  51
  52 #define NUM_PACKETS (1 << 18)
  53 #define MAX_EVENTS  (16 * 1024)
  54
  55 static int evdev;
  56 static struct rte_mempool *eventdev_test_mempool;
  57
  58 struct event_attr {
  59         uint32_t flow_id;
  60         uint8_t event_type;
  61         uint8_t sub_event_type;
  62         uint8_t sched_type;
  63         uint8_t queue;
  64         uint8_t port;
  65 };
  66
  67 static uint32_t seqn_list_index;
  68 static int seqn_list[NUM_PACKETS];
  69
  70 static inline void
  71 seqn_list_init(void)
  72 {
  73         RTE_BUILD_BUG_ON(NUM_PACKETS < MAX_EVENTS);
  74         memset(seqn_list, 0, sizeof(seqn_list));
  75         seqn_list_index = 0;
  76 }
  77
  78 static inline int
  79 seqn_list_update(int val)
  80 {
  81         if (seqn_list_index >= NUM_PACKETS)
  82                 return TEST_FAILED;
  83
  84         seqn_list[seqn_list_index++] = val;
  85         rte_smp_wmb();
  86         return TEST_SUCCESS;
  87 }
  88
  89 static inline int
  90 seqn_list_check(int limit)
  91 {
  92         int i;
  93
  94         for (i = 0; i < limit; i++) {
  95                 if (seqn_list[i] != i) {
  96                         printf("Seqn mismatch %d %d\n", seqn_list[i], i);
  97                         return TEST_FAILED;
  98                 }
  99         }
 100         return TEST_SUCCESS;
 101 }
 102
 103 struct test_core_param {
 104         rte_atomic32_t *total_events;
 105         uint64_t dequeue_tmo_ticks;
 106         uint8_t port;
 107         uint8_t sched_type;
 108 };
 109
 110 static int
 111 testsuite_setup(void)
 112 {
 113         const char *eventdev_name = "event_octeontx";
 114
 115         evdev = rte_event_dev_get_dev_id(eventdev_name);
 116         if (evdev < 0) {
 117                 printf("%d: Eventdev %s not found - creating.\n",
 118                                 __LINE__, eventdev_name);
 119                 if (rte_vdev_init(eventdev_name, NULL) < 0) {
 120                         printf("Error creating eventdev %s\n", eventdev_name);
 121                         return TEST_FAILED;
 122                 }
 123                 evdev = rte_event_dev_get_dev_id(eventdev_name);
 124                 if (evdev < 0) {
 125                         printf("Error finding newly created eventdev\n");
 126                         return TEST_FAILED;
 127                 }
 128         }
 129
 130         return TEST_SUCCESS;
 131 }
 132
 133 static void
 134 testsuite_teardown(void)
 135 {
 136         rte_event_dev_close(evdev);
 137 }
 138
 139 static inline void
 140 devconf_set_default_sane_values(struct rte_event_dev_config *dev_conf,
 141                         struct rte_event_dev_info *info)
 142 {
 143         memset(dev_conf, 0, sizeof(struct rte_event_dev_config));
 144         dev_conf->dequeue_timeout_ns = info->min_dequeue_timeout_ns;
 145         dev_conf->nb_event_ports = info->max_event_ports;
 146         dev_conf->nb_event_queues = info->max_event_queues;
 147         dev_conf->nb_event_queue_flows = info->max_event_queue_flows;
 148         dev_conf->nb_event_port_dequeue_depth =
 149                         info->max_event_port_dequeue_depth;
 150         dev_conf->nb_event_port_enqueue_depth =
 151                         info->max_event_port_enqueue_depth;
 152         dev_conf->nb_event_port_enqueue_depth =
 153                         info->max_event_port_enqueue_depth;
 154         dev_conf->nb_events_limit =
 155                         info->max_num_events;
 156 }
 157
 158 enum {
 159         TEST_EVENTDEV_SETUP_DEFAULT,
 160         TEST_EVENTDEV_SETUP_PRIORITY,
 161         TEST_EVENTDEV_SETUP_DEQUEUE_TIMEOUT,
 162 };
 163
 164 static inline int
 165 _eventdev_setup(int mode)
 166 {
 167         int i, ret;
 168         struct rte_event_dev_config dev_conf;
 169         struct rte_event_dev_info info;
 170         const char *pool_name = "evdev_octeontx_test_pool";
 171
 172         /* Create and destrory pool for each test case to make it standalone */
 173         eventdev_test_mempool = rte_pktmbuf_pool_create(pool_name,
 174                                         MAX_EVENTS,
 175                                         0 /*MBUF_CACHE_SIZE*/,
 176                                         0,
 177                                         512, /* Use very small mbufs */
 178                                         rte_socket_id());
 179         if (!eventdev_test_mempool) {
 180                 printf("ERROR creating mempool\n");
 181                 return TEST_FAILED;
 182         }
 183
 184         ret = rte_event_dev_info_get(evdev, &info);
 185         TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info");
 186         TEST_ASSERT(info.max_num_events >= (int32_t)MAX_EVENTS,
 187                         "max_num_events=%d < max_events=%d",
 188                         info.max_num_events, MAX_EVENTS);
 189
 190         devconf_set_default_sane_values(&dev_conf, &info);
 191         if (mode == TEST_EVENTDEV_SETUP_DEQUEUE_TIMEOUT)
 192                 dev_conf.event_dev_cfg |= RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT;
 193
 194         ret = rte_event_dev_configure(evdev, &dev_conf);
 195         TEST_ASSERT_SUCCESS(ret, "Failed to configure eventdev");
 196
 197         uint32_t queue_count;
 198         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 199                             RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 200                             &queue_count), "Queue count get failed");
 201
 202         if (mode == TEST_EVENTDEV_SETUP_PRIORITY) {
 203                 if (queue_count > 8) {
 204                         printf("test expects the unique priority per queue\n");
 205                         return -ENOTSUP;
 206                 }
 207
 208                 /* Configure event queues(0 to n) with
 209                  * RTE_EVENT_DEV_PRIORITY_HIGHEST to
 210                  * RTE_EVENT_DEV_PRIORITY_LOWEST
 211                  */
 212                 uint8_t step = (RTE_EVENT_DEV_PRIORITY_LOWEST + 1) /
 213                                 queue_count;
 214                 for (i = 0; i < (int)queue_count; i++) {
 215                         struct rte_event_queue_conf queue_conf;
 216
 217                         ret = rte_event_queue_default_conf_get(evdev, i,
 218                                                 &queue_conf);
 219                         TEST_ASSERT_SUCCESS(ret, "Failed to get def_conf%d", i);
 220                         queue_conf.priority = i * step;
 221                         ret = rte_event_queue_setup(evdev, i, &queue_conf);
 222                         TEST_ASSERT_SUCCESS(ret, "Failed to setup queue=%d", i);
 223                 }
 224
 225         } else {
 226                 /* Configure event queues with default priority */
 227                 for (i = 0; i < (int)queue_count; i++) {
 228                         ret = rte_event_queue_setup(evdev, i, NULL);
 229                         TEST_ASSERT_SUCCESS(ret, "Failed to setup queue=%d", i);
 230                 }
 231         }
 232         /* Configure event ports */
 233         uint32_t port_count;
 234         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 235                                 RTE_EVENT_DEV_ATTR_PORT_COUNT,
 236                                 &port_count), "Port count get failed");
 237         for (i = 0; i < (int)port_count; i++) {
 238                 ret = rte_event_port_setup(evdev, i, NULL);
 239                 TEST_ASSERT_SUCCESS(ret, "Failed to setup port=%d", i);
 240                 ret = rte_event_port_link(evdev, i, NULL, NULL, 0);
 241                 TEST_ASSERT(ret >= 0, "Failed to link all queues port=%d", i);
 242         }
 243
 244         ret = rte_event_dev_start(evdev);
 245         TEST_ASSERT_SUCCESS(ret, "Failed to start device");
 246
 247         return TEST_SUCCESS;
 248 }
 249
 250 static inline int
 251 eventdev_setup(void)
 252 {
 253         return _eventdev_setup(TEST_EVENTDEV_SETUP_DEFAULT);
 254 }
 255
 256 static inline int
 257 eventdev_setup_priority(void)
 258 {
 259         return _eventdev_setup(TEST_EVENTDEV_SETUP_PRIORITY);
 260 }
 261
 262 static inline int
 263 eventdev_setup_dequeue_timeout(void)
 264 {
 265         return _eventdev_setup(TEST_EVENTDEV_SETUP_DEQUEUE_TIMEOUT);
 266 }
 267
 268 static inline void
 269 eventdev_teardown(void)
 270 {
 271         rte_event_dev_stop(evdev);
 272         rte_mempool_free(eventdev_test_mempool);
 273 }
 274
 275 static inline void
 276 update_event_and_validation_attr(struct rte_mbuf *m, struct rte_event *ev,
 277                         uint32_t flow_id, uint8_t event_type,
 278                         uint8_t sub_event_type, uint8_t sched_type,
 279                         uint8_t queue, uint8_t port)
 280 {
 281         struct event_attr *attr;
 282
 283         /* Store the event attributes in mbuf for future reference */
 284         attr = rte_pktmbuf_mtod(m, struct event_attr *);
 285         attr->flow_id = flow_id;
 286         attr->event_type = event_type;
 287         attr->sub_event_type = sub_event_type;
 288         attr->sched_type = sched_type;
 289         attr->queue = queue;
 290         attr->port = port;
 291
 292         ev->flow_id = flow_id;
 293         ev->sub_event_type = sub_event_type;
 294         ev->event_type = event_type;
 295         /* Inject the new event */
 296         ev->op = RTE_EVENT_OP_NEW;
 297         ev->sched_type = sched_type;
 298         ev->queue_id = queue;
 299         ev->mbuf = m;
 300 }
 301
 302 static inline int
 303 inject_events(uint32_t flow_id, uint8_t event_type, uint8_t sub_event_type,
 304                 uint8_t sched_type, uint8_t queue, uint8_t port,
 305                 unsigned int events)
 306 {
 307         struct rte_mbuf *m;
 308         unsigned int i;
 309
 310         for (i = 0; i < events; i++) {
 311                 struct rte_event ev = {.event = 0, .u64 = 0};
 312
 313                 m = rte_pktmbuf_alloc(eventdev_test_mempool);
 314                 TEST_ASSERT_NOT_NULL(m, "mempool alloc failed");
 315
 316                 m->seqn = i;
 317                 update_event_and_validation_attr(m, &ev, flow_id, event_type,
 318                         sub_event_type, sched_type, queue, port);
 319                 rte_event_enqueue_burst(evdev, port, &ev, 1);
 320         }
 321         return 0;
 322 }
 323
 324 static inline int
 325 check_excess_events(uint8_t port)
 326 {
 327         int i;
 328         uint16_t valid_event;
 329         struct rte_event ev;
 330
 331         /* Check for excess events, try for a few times and exit */
 332         for (i = 0; i < 32; i++) {
 333                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
 334
 335                 TEST_ASSERT_SUCCESS(valid_event, "Unexpected valid event=%d",
 336                                         ev.mbuf->seqn);
 337         }
 338         return 0;
 339 }
 340
 341 static inline int
 342 generate_random_events(const unsigned int total_events)
 343 {
 344         struct rte_event_dev_info info;
 345         unsigned int i;
 346         int ret;
 347
 348         uint32_t queue_count;
 349         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 350                             RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 351                             &queue_count), "Queue count get failed");
 352
 353         ret = rte_event_dev_info_get(evdev, &info);
 354         TEST_ASSERT_SUCCESS(ret, "Failed to get event dev info");
 355         for (i = 0; i < total_events; i++) {
 356                 ret = inject_events(
 357                         rte_rand() % info.max_event_queue_flows /*flow_id */,
 358                         RTE_EVENT_TYPE_CPU /* event_type */,
 359                         rte_rand() % 256 /* sub_event_type */,
 360                         rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1),
 361                         rte_rand() % queue_count /* queue */,
 362                         0 /* port */,
 363                         1 /* events */);
 364                 if (ret)
 365                         return TEST_FAILED;
 366         }
 367         return ret;
 368 }
 369
 370
 371 static inline int
 372 validate_event(struct rte_event *ev)
 373 {
 374         struct event_attr *attr;
 375
 376         attr = rte_pktmbuf_mtod(ev->mbuf, struct event_attr *);
 377         TEST_ASSERT_EQUAL(attr->flow_id, ev->flow_id,
 378                         "flow_id mismatch enq=%d deq =%d",
 379                         attr->flow_id, ev->flow_id);
 380         TEST_ASSERT_EQUAL(attr->event_type, ev->event_type,
 381                         "event_type mismatch enq=%d deq =%d",
 382                         attr->event_type, ev->event_type);
 383         TEST_ASSERT_EQUAL(attr->sub_event_type, ev->sub_event_type,
 384                         "sub_event_type mismatch enq=%d deq =%d",
 385                         attr->sub_event_type, ev->sub_event_type);
 386         TEST_ASSERT_EQUAL(attr->sched_type, ev->sched_type,
 387                         "sched_type mismatch enq=%d deq =%d",
 388                         attr->sched_type, ev->sched_type);
 389         TEST_ASSERT_EQUAL(attr->queue, ev->queue_id,
 390                         "queue mismatch enq=%d deq =%d",
 391                         attr->queue, ev->queue_id);
 392         return 0;
 393 }
 394
 395 typedef int (*validate_event_cb)(uint32_t index, uint8_t port,
 396                                  struct rte_event *ev);
 397
 398 static inline int
 399 consume_events(uint8_t port, const uint32_t total_events, validate_event_cb fn)
 400 {
 401         int ret;
 402         uint16_t valid_event;
 403         uint32_t events = 0, forward_progress_cnt = 0, index = 0;
 404         struct rte_event ev;
 405
 406         while (1) {
 407                 if (++forward_progress_cnt > UINT16_MAX) {
 408                         printf("Detected deadlock\n");
 409                         return TEST_FAILED;
 410                 }
 411
 412                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
 413                 if (!valid_event)
 414                         continue;
 415
 416                 forward_progress_cnt = 0;
 417                 ret = validate_event(&ev);
 418                 if (ret)
 419                         return TEST_FAILED;
 420
 421                 if (fn != NULL) {
 422                         ret = fn(index, port, &ev);
 423                         TEST_ASSERT_SUCCESS(ret,
 424                                 "Failed to validate test specific event");
 425                 }
 426
 427                 ++index;
 428
 429                 rte_pktmbuf_free(ev.mbuf);
 430                 if (++events >= total_events)
 431                         break;
 432         }
 433
 434         return check_excess_events(port);
 435 }
 436
 437 static int
 438 validate_simple_enqdeq(uint32_t index, uint8_t port, struct rte_event *ev)
 439 {
 440         RTE_SET_USED(port);
 441         TEST_ASSERT_EQUAL(index, ev->mbuf->seqn, "index=%d != seqn=%d", index,
 442                                         ev->mbuf->seqn);
 443         return 0;
 444 }
 445
 446 static inline int
 447 test_simple_enqdeq(uint8_t sched_type)
 448 {
 449         int ret;
 450
 451         ret = inject_events(0 /*flow_id */,
 452                                 RTE_EVENT_TYPE_CPU /* event_type */,
 453                                 0 /* sub_event_type */,
 454                                 sched_type,
 455                                 0 /* queue */,
 456                                 0 /* port */,
 457                                 MAX_EVENTS);
 458         if (ret)
 459                 return TEST_FAILED;
 460
 461         return consume_events(0 /* port */, MAX_EVENTS, validate_simple_enqdeq);
 462 }
 463
 464 static int
 465 test_simple_enqdeq_ordered(void)
 466 {
 467         return test_simple_enqdeq(RTE_SCHED_TYPE_ORDERED);
 468 }
 469
 470 static int
 471 test_simple_enqdeq_atomic(void)
 472 {
 473         return test_simple_enqdeq(RTE_SCHED_TYPE_ATOMIC);
 474 }
 475
 476 static int
 477 test_simple_enqdeq_parallel(void)
 478 {
 479         return test_simple_enqdeq(RTE_SCHED_TYPE_PARALLEL);
 480 }
 481
 482 /*
 483  * Generate a prescribed number of events and spread them across available
 484  * queues. On dequeue, using single event port(port 0) verify the enqueued
 485  * event attributes
 486  */
 487 static int
 488 test_multi_queue_enq_single_port_deq(void)
 489 {
 490         int ret;
 491
 492         ret = generate_random_events(MAX_EVENTS);
 493         if (ret)
 494                 return TEST_FAILED;
 495
 496         return consume_events(0 /* port */, MAX_EVENTS, NULL);
 497 }
 498
 499 /*
 500  * Inject 0..MAX_EVENTS events over 0..queue_count with modulus
 501  * operation
 502  *
 503  * For example, Inject 32 events over 0..7 queues
 504  * enqueue events 0, 8, 16, 24 in queue 0
 505  * enqueue events 1, 9, 17, 25 in queue 1
 506  * ..
 507  * ..
 508  * enqueue events 7, 15, 23, 31 in queue 7
 509  *
 510  * On dequeue, Validate the events comes in 0,8,16,24,1,9,17,25..,7,15,23,31
 511  * order from queue0(highest priority) to queue7(lowest_priority)
 512  */
 513 static int
 514 validate_queue_priority(uint32_t index, uint8_t port, struct rte_event *ev)
 515 {
 516         uint32_t queue_count;
 517         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 518                             RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 519                             &queue_count), "Queue count get failed");
 520         uint32_t range = MAX_EVENTS / queue_count;
 521         uint32_t expected_val = (index % range) * queue_count;
 522
 523         expected_val += ev->queue_id;
 524         RTE_SET_USED(port);
 525         TEST_ASSERT_EQUAL(ev->mbuf->seqn, expected_val,
 526         "seqn=%d index=%d expected=%d range=%d nb_queues=%d max_event=%d",
 527                         ev->mbuf->seqn, index, expected_val, range,
 528                         queue_count, MAX_EVENTS);
 529         return 0;
 530 }
 531
 532 static int
 533 test_multi_queue_priority(void)
 534 {
 535         uint8_t queue;
 536         struct rte_mbuf *m;
 537         int i, max_evts_roundoff;
 538
 539         /* See validate_queue_priority() comments for priority validate logic */
 540         uint32_t queue_count;
 541         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 542                             RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 543                             &queue_count), "Queue count get failed");
 544         max_evts_roundoff  = MAX_EVENTS / queue_count;
 545         max_evts_roundoff *= queue_count;
 546
 547         for (i = 0; i < max_evts_roundoff; i++) {
 548                 struct rte_event ev = {.event = 0, .u64 = 0};
 549
 550                 m = rte_pktmbuf_alloc(eventdev_test_mempool);
 551                 TEST_ASSERT_NOT_NULL(m, "mempool alloc failed");
 552
 553                 m->seqn = i;
 554                 queue = i % queue_count;
 555                 update_event_and_validation_attr(m, &ev, 0, RTE_EVENT_TYPE_CPU,
 556                         0, RTE_SCHED_TYPE_PARALLEL, queue, 0);
 557                 rte_event_enqueue_burst(evdev, 0, &ev, 1);
 558         }
 559
 560         return consume_events(0, max_evts_roundoff, validate_queue_priority);
 561 }
 562
 563 static int
 564 worker_multi_port_fn(void *arg)
 565 {
 566         struct test_core_param *param = arg;
 567         struct rte_event ev;
 568         uint16_t valid_event;
 569         uint8_t port = param->port;
 570         rte_atomic32_t *total_events = param->total_events;
 571         int ret;
 572
 573         while (rte_atomic32_read(total_events) > 0) {
 574                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
 575                 if (!valid_event)
 576                         continue;
 577
 578                 ret = validate_event(&ev);
 579                 TEST_ASSERT_SUCCESS(ret, "Failed to validate event");
 580                 rte_pktmbuf_free(ev.mbuf);
 581                 rte_atomic32_sub(total_events, 1);
 582         }
 583         return 0;
 584 }
 585
 586 static inline int
 587 wait_workers_to_join(int lcore, const rte_atomic32_t *count)
 588 {
 589         uint64_t cycles, print_cycles;
 590
 591         print_cycles = cycles = rte_get_timer_cycles();
 592         while (rte_eal_get_lcore_state(lcore) != FINISHED) {
 593                 uint64_t new_cycles = rte_get_timer_cycles();
 594
 595                 if (new_cycles - print_cycles > rte_get_timer_hz()) {
 596                         printf("\r%s: events %d\n", __func__,
 597                                 rte_atomic32_read(count));
 598                         print_cycles = new_cycles;
 599                 }
 600                 if (new_cycles - cycles > rte_get_timer_hz() * 10) {
 601                         printf("%s: No schedules for seconds, deadlock (%d)\n",
 602                                 __func__,
 603                                 rte_atomic32_read(count));
 604                         rte_event_dev_dump(evdev, stdout);
 605                         cycles = new_cycles;
 606                         return TEST_FAILED;
 607                 }
 608         }
 609         rte_eal_mp_wait_lcore();
 610         return TEST_SUCCESS;
 611 }
 612
 613
 614 static inline int
 615 launch_workers_and_wait(int (*master_worker)(void *),
 616                         int (*slave_workers)(void *), uint32_t total_events,
 617                         uint8_t nb_workers, uint8_t sched_type)
 618 {
 619         uint8_t port = 0;
 620         int w_lcore;
 621         int ret;
 622         struct test_core_param *param;
 623         rte_atomic32_t atomic_total_events;
 624         uint64_t dequeue_tmo_ticks;
 625
 626         if (!nb_workers)
 627                 return 0;
 628
 629         rte_atomic32_set(&atomic_total_events, total_events);
 630         seqn_list_init();
 631
 632         param = malloc(sizeof(struct test_core_param) * nb_workers);
 633         if (!param)
 634                 return TEST_FAILED;
 635
 636         ret = rte_event_dequeue_timeout_ticks(evdev,
 637                 rte_rand() % 10000000/* 10ms */, &dequeue_tmo_ticks);
 638         if (ret)
 639                 return TEST_FAILED;
 640
 641         param[0].total_events = &atomic_total_events;
 642         param[0].sched_type = sched_type;
 643         param[0].port = 0;
 644         param[0].dequeue_tmo_ticks = dequeue_tmo_ticks;
 645         rte_smp_wmb();
 646
 647         w_lcore = rte_get_next_lcore(
 648                         /* start core */ -1,
 649                         /* skip master */ 1,
 650                         /* wrap */ 0);
 651         rte_eal_remote_launch(master_worker, &param[0], w_lcore);
 652
 653         for (port = 1; port < nb_workers; port++) {
 654                 param[port].total_events = &atomic_total_events;
 655                 param[port].sched_type = sched_type;
 656                 param[port].port = port;
 657                 param[port].dequeue_tmo_ticks = dequeue_tmo_ticks;
 658                 rte_smp_wmb();
 659                 w_lcore = rte_get_next_lcore(w_lcore, 1, 0);
 660                 rte_eal_remote_launch(slave_workers, &param[port], w_lcore);
 661         }
 662
 663         ret = wait_workers_to_join(w_lcore, &atomic_total_events);
 664         free(param);
 665         return ret;
 666 }
 667
 668 /*
 669  * Generate a prescribed number of events and spread them across available
 670  * queues. Dequeue the events through multiple ports and verify the enqueued
 671  * event attributes
 672  */
 673 static int
 674 test_multi_queue_enq_multi_port_deq(void)
 675 {
 676         const unsigned int total_events = MAX_EVENTS;
 677         uint32_t nr_ports;
 678         int ret;
 679
 680         ret = generate_random_events(total_events);
 681         if (ret)
 682                 return TEST_FAILED;
 683
 684         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 685                                 RTE_EVENT_DEV_ATTR_PORT_COUNT,
 686                                 &nr_ports), "Port count get failed");
 687         nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1);
 688
 689         if (!nr_ports) {
 690                 printf("%s: Not enough ports=%d or workers=%d\n", __func__,
 691                         nr_ports, rte_lcore_count() - 1);
 692                 return TEST_SUCCESS;
 693         }
 694
 695         return launch_workers_and_wait(worker_multi_port_fn,
 696                                         worker_multi_port_fn, total_events,
 697                                         nr_ports, 0xff /* invalid */);
 698 }
 699
 700 static int
 701 validate_queue_to_port_single_link(uint32_t index, uint8_t port,
 702                         struct rte_event *ev)
 703 {
 704         RTE_SET_USED(index);
 705         TEST_ASSERT_EQUAL(port, ev->queue_id,
 706                                 "queue mismatch enq=%d deq =%d",
 707                                 port, ev->queue_id);
 708         return 0;
 709 }
 710
 711 /*
 712  * Link queue x to port x and check correctness of link by checking
 713  * queue_id == x on dequeue on the specific port x
 714  */
 715 static int
 716 test_queue_to_port_single_link(void)
 717 {
 718         int i, nr_links, ret;
 719
 720         uint32_t port_count;
 721         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 722                                 RTE_EVENT_DEV_ATTR_PORT_COUNT,
 723                                 &port_count), "Port count get failed");
 724
 725         /* Unlink all connections that created in eventdev_setup */
 726         for (i = 0; i < (int)port_count; i++) {
 727                 ret = rte_event_port_unlink(evdev, i, NULL, 0);
 728                 TEST_ASSERT(ret >= 0, "Failed to unlink all queues port=%d", i);
 729         }
 730
 731         uint32_t queue_count;
 732         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 733                             RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 734                             &queue_count), "Queue count get failed");
 735
 736         nr_links = RTE_MIN(port_count, queue_count);
 737         const unsigned int total_events = MAX_EVENTS / nr_links;
 738
 739         /* Link queue x to port x and inject events to queue x through port x */
 740         for (i = 0; i < nr_links; i++) {
 741                 uint8_t queue = (uint8_t)i;
 742
 743                 ret = rte_event_port_link(evdev, i, &queue, NULL, 1);
 744                 TEST_ASSERT(ret == 1, "Failed to link queue to port %d", i);
 745
 746                 ret = inject_events(
 747                         0x100 /*flow_id */,
 748                         RTE_EVENT_TYPE_CPU /* event_type */,
 749                         rte_rand() % 256 /* sub_event_type */,
 750                         rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1),
 751                         queue /* queue */,
 752                         i /* port */,
 753                         total_events /* events */);
 754                 if (ret)
 755                         return TEST_FAILED;
 756         }
 757
 758         /* Verify the events generated from correct queue */
 759         for (i = 0; i < nr_links; i++) {
 760                 ret = consume_events(i /* port */, total_events,
 761                                 validate_queue_to_port_single_link);
 762                 if (ret)
 763                         return TEST_FAILED;
 764         }
 765
 766         return TEST_SUCCESS;
 767 }
 768
 769 static int
 770 validate_queue_to_port_multi_link(uint32_t index, uint8_t port,
 771                         struct rte_event *ev)
 772 {
 773         RTE_SET_USED(index);
 774         TEST_ASSERT_EQUAL(port, (ev->queue_id & 0x1),
 775                                 "queue mismatch enq=%d deq =%d",
 776                                 port, ev->queue_id);
 777         return 0;
 778 }
 779
 780 /*
 781  * Link all even number of queues to port 0 and all odd number of queues to
 782  * port 1 and verify the link connection on dequeue
 783  */
 784 static int
 785 test_queue_to_port_multi_link(void)
 786 {
 787         int ret, port0_events = 0, port1_events = 0;
 788         uint8_t queue, port;
 789         uint32_t nr_queues = 0;
 790         uint32_t nr_ports = 0;
 791
 792         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 793                             RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 794                             &nr_queues), "Queue count get failed");
 795
 796         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 797                                 RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
 798                                 &nr_queues), "Queue count get failed");
 799         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 800                                 RTE_EVENT_DEV_ATTR_PORT_COUNT,
 801                                 &nr_ports), "Port count get failed");
 802
 803         if (nr_ports < 2) {
 804                 printf("%s: Not enough ports to test ports=%d\n",
 805                                 __func__, nr_ports);
 806                 return TEST_SUCCESS;
 807         }
 808
 809         /* Unlink all connections that created in eventdev_setup */
 810         for (port = 0; port < nr_ports; port++) {
 811                 ret = rte_event_port_unlink(evdev, port, NULL, 0);
 812                 TEST_ASSERT(ret >= 0, "Failed to unlink all queues port=%d",
 813                                         port);
 814         }
 815
 816         const unsigned int total_events = MAX_EVENTS / nr_queues;
 817
 818         /* Link all even number of queues to port0 and odd numbers to port 1*/
 819         for (queue = 0; queue < nr_queues; queue++) {
 820                 port = queue & 0x1;
 821                 ret = rte_event_port_link(evdev, port, &queue, NULL, 1);
 822                 TEST_ASSERT(ret == 1, "Failed to link queue=%d to port=%d",
 823                                         queue, port);
 824
 825                 ret = inject_events(
 826                         0x100 /*flow_id */,
 827                         RTE_EVENT_TYPE_CPU /* event_type */,
 828                         rte_rand() % 256 /* sub_event_type */,
 829                         rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1),
 830                         queue /* queue */,
 831                         port /* port */,
 832                         total_events /* events */);
 833                 if (ret)
 834                         return TEST_FAILED;
 835
 836                 if (port == 0)
 837                         port0_events += total_events;
 838                 else
 839                         port1_events += total_events;
 840         }
 841
 842         ret = consume_events(0 /* port */, port0_events,
 843                                 validate_queue_to_port_multi_link);
 844         if (ret)
 845                 return TEST_FAILED;
 846         ret = consume_events(1 /* port */, port1_events,
 847                                 validate_queue_to_port_multi_link);
 848         if (ret)
 849                 return TEST_FAILED;
 850
 851         return TEST_SUCCESS;
 852 }
 853
 854 static int
 855 worker_flow_based_pipeline(void *arg)
 856 {
 857         struct test_core_param *param = arg;
 858         struct rte_event ev;
 859         uint16_t valid_event;
 860         uint8_t port = param->port;
 861         uint8_t new_sched_type = param->sched_type;
 862         rte_atomic32_t *total_events = param->total_events;
 863         uint64_t dequeue_tmo_ticks = param->dequeue_tmo_ticks;
 864
 865         while (rte_atomic32_read(total_events) > 0) {
 866                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1,
 867                                         dequeue_tmo_ticks);
 868                 if (!valid_event)
 869                         continue;
 870
 871                 /* Events from stage 0 */
 872                 if (ev.sub_event_type == 0) {
 873                         /* Move to atomic flow to maintain the ordering */
 874                         ev.flow_id = 0x2;
 875                         ev.event_type = RTE_EVENT_TYPE_CPU;
 876                         ev.sub_event_type = 1; /* stage 1 */
 877                         ev.sched_type = new_sched_type;
 878                         ev.op = RTE_EVENT_OP_FORWARD;
 879                         rte_event_enqueue_burst(evdev, port, &ev, 1);
 880                 } else if (ev.sub_event_type == 1) { /* Events from stage 1*/
 881                         if (seqn_list_update(ev.mbuf->seqn) == TEST_SUCCESS) {
 882                                 rte_pktmbuf_free(ev.mbuf);
 883                                 rte_atomic32_sub(total_events, 1);
 884                         } else {
 885                                 printf("Failed to update seqn_list\n");
 886                                 return TEST_FAILED;
 887                         }
 888                 } else {
 889                         printf("Invalid ev.sub_event_type = %d\n",
 890                                         ev.sub_event_type);
 891                         return TEST_FAILED;
 892                 }
 893         }
 894         return 0;
 895 }
 896
 897 static int
 898 test_multiport_flow_sched_type_test(uint8_t in_sched_type,
 899                         uint8_t out_sched_type)
 900 {
 901         const unsigned int total_events = MAX_EVENTS;
 902         uint32_t nr_ports;
 903         int ret;
 904
 905         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
 906                                 RTE_EVENT_DEV_ATTR_PORT_COUNT,
 907                                 &nr_ports), "Port count get failed");
 908         nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1);
 909
 910         if (!nr_ports) {
 911                 printf("%s: Not enough ports=%d or workers=%d\n", __func__,
 912                         nr_ports, rte_lcore_count() - 1);
 913                 return TEST_SUCCESS;
 914         }
 915
 916         /* Injects events with m->seqn=0 to total_events */
 917         ret = inject_events(
 918                 0x1 /*flow_id */,
 919                 RTE_EVENT_TYPE_CPU /* event_type */,
 920                 0 /* sub_event_type (stage 0) */,
 921                 in_sched_type,
 922                 0 /* queue */,
 923                 0 /* port */,
 924                 total_events /* events */);
 925         if (ret)
 926                 return TEST_FAILED;
 927
 928         ret = launch_workers_and_wait(worker_flow_based_pipeline,
 929                                         worker_flow_based_pipeline,
 930                                         total_events, nr_ports, out_sched_type);
 931         if (ret)
 932                 return TEST_FAILED;
 933
 934         if (in_sched_type != RTE_SCHED_TYPE_PARALLEL &&
 935                         out_sched_type == RTE_SCHED_TYPE_ATOMIC) {
 936                 /* Check the events order maintained or not */
 937                 return seqn_list_check(total_events);
 938         }
 939         return TEST_SUCCESS;
 940 }
 941
 942
 943 /* Multi port ordered to atomic transaction */
 944 static int
 945 test_multi_port_flow_ordered_to_atomic(void)
 946 {
 947         /* Ingress event order test */
 948         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ORDERED,
 949                                 RTE_SCHED_TYPE_ATOMIC);
 950 }
 951
 952 static int
 953 test_multi_port_flow_ordered_to_ordered(void)
 954 {
 955         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ORDERED,
 956                                 RTE_SCHED_TYPE_ORDERED);
 957 }
 958
 959 static int
 960 test_multi_port_flow_ordered_to_parallel(void)
 961 {
 962         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ORDERED,
 963                                 RTE_SCHED_TYPE_PARALLEL);
 964 }
 965
 966 static int
 967 test_multi_port_flow_atomic_to_atomic(void)
 968 {
 969         /* Ingress event order test */
 970         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
 971                                 RTE_SCHED_TYPE_ATOMIC);
 972 }
 973
 974 static int
 975 test_multi_port_flow_atomic_to_ordered(void)
 976 {
 977         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
 978                                 RTE_SCHED_TYPE_ORDERED);
 979 }
 980
 981 static int
 982 test_multi_port_flow_atomic_to_parallel(void)
 983 {
 984         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
 985                                 RTE_SCHED_TYPE_PARALLEL);
 986 }
 987
 988 static int
 989 test_multi_port_flow_parallel_to_atomic(void)
 990 {
 991         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
 992                                 RTE_SCHED_TYPE_ATOMIC);
 993 }
 994
 995 static int
 996 test_multi_port_flow_parallel_to_ordered(void)
 997 {
 998         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
 999                                 RTE_SCHED_TYPE_ORDERED);
1000 }
1001
1002 static int
1003 test_multi_port_flow_parallel_to_parallel(void)
1004 {
1005         return test_multiport_flow_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
1006                                 RTE_SCHED_TYPE_PARALLEL);
1007 }
1008
1009 static int
1010 worker_group_based_pipeline(void *arg)
1011 {
1012         struct test_core_param *param = arg;
1013         struct rte_event ev;
1014         uint16_t valid_event;
1015         uint8_t port = param->port;
1016         uint8_t new_sched_type = param->sched_type;
1017         rte_atomic32_t *total_events = param->total_events;
1018         uint64_t dequeue_tmo_ticks = param->dequeue_tmo_ticks;
1019
1020         while (rte_atomic32_read(total_events) > 0) {
1021                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1,
1022                                         dequeue_tmo_ticks);
1023                 if (!valid_event)
1024                         continue;
1025
1026                 /* Events from stage 0(group 0) */
1027                 if (ev.queue_id == 0) {
1028                         /* Move to atomic flow to maintain the ordering */
1029                         ev.flow_id = 0x2;
1030                         ev.event_type = RTE_EVENT_TYPE_CPU;
1031                         ev.sched_type = new_sched_type;
1032                         ev.queue_id = 1; /* Stage 1*/
1033                         ev.op = RTE_EVENT_OP_FORWARD;
1034                         rte_event_enqueue_burst(evdev, port, &ev, 1);
1035                 } else if (ev.queue_id == 1) { /* Events from stage 1(group 1)*/
1036                         if (seqn_list_update(ev.mbuf->seqn) == TEST_SUCCESS) {
1037                                 rte_pktmbuf_free(ev.mbuf);
1038                                 rte_atomic32_sub(total_events, 1);
1039                         } else {
1040                                 printf("Failed to update seqn_list\n");
1041                                 return TEST_FAILED;
1042                         }
1043                 } else {
1044                         printf("Invalid ev.queue_id = %d\n", ev.queue_id);
1045                         return TEST_FAILED;
1046                 }
1047         }
1048
1049
1050         return 0;
1051 }
1052
1053 static int
1054 test_multiport_queue_sched_type_test(uint8_t in_sched_type,
1055                         uint8_t out_sched_type)
1056 {
1057         const unsigned int total_events = MAX_EVENTS;
1058         uint32_t nr_ports;
1059         int ret;
1060
1061         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
1062                                 RTE_EVENT_DEV_ATTR_PORT_COUNT,
1063                                 &nr_ports), "Port count get failed");
1064
1065         nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1);
1066
1067         uint32_t queue_count;
1068         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
1069                             RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
1070                             &queue_count), "Queue count get failed");
1071         if (queue_count < 2 ||  !nr_ports) {
1072                 printf("%s: Not enough queues=%d ports=%d or workers=%d\n",
1073                          __func__, queue_count, nr_ports,
1074                          rte_lcore_count() - 1);
1075                 return TEST_SUCCESS;
1076         }
1077
1078         /* Injects events with m->seqn=0 to total_events */
1079         ret = inject_events(
1080                 0x1 /*flow_id */,
1081                 RTE_EVENT_TYPE_CPU /* event_type */,
1082                 0 /* sub_event_type (stage 0) */,
1083                 in_sched_type,
1084                 0 /* queue */,
1085                 0 /* port */,
1086                 total_events /* events */);
1087         if (ret)
1088                 return TEST_FAILED;
1089
1090         ret = launch_workers_and_wait(worker_group_based_pipeline,
1091                                         worker_group_based_pipeline,
1092                                         total_events, nr_ports, out_sched_type);
1093         if (ret)
1094                 return TEST_FAILED;
1095
1096         if (in_sched_type != RTE_SCHED_TYPE_PARALLEL &&
1097                         out_sched_type == RTE_SCHED_TYPE_ATOMIC) {
1098                 /* Check the events order maintained or not */
1099                 return seqn_list_check(total_events);
1100         }
1101         return TEST_SUCCESS;
1102 }
1103
1104 static int
1105 test_multi_port_queue_ordered_to_atomic(void)
1106 {
1107         /* Ingress event order test */
1108         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ORDERED,
1109                                 RTE_SCHED_TYPE_ATOMIC);
1110 }
1111
1112 static int
1113 test_multi_port_queue_ordered_to_ordered(void)
1114 {
1115         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ORDERED,
1116                                 RTE_SCHED_TYPE_ORDERED);
1117 }
1118
1119 static int
1120 test_multi_port_queue_ordered_to_parallel(void)
1121 {
1122         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ORDERED,
1123                                 RTE_SCHED_TYPE_PARALLEL);
1124 }
1125
1126 static int
1127 test_multi_port_queue_atomic_to_atomic(void)
1128 {
1129         /* Ingress event order test */
1130         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
1131                                 RTE_SCHED_TYPE_ATOMIC);
1132 }
1133
1134 static int
1135 test_multi_port_queue_atomic_to_ordered(void)
1136 {
1137         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
1138                                 RTE_SCHED_TYPE_ORDERED);
1139 }
1140
1141 static int
1142 test_multi_port_queue_atomic_to_parallel(void)
1143 {
1144         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_ATOMIC,
1145                                 RTE_SCHED_TYPE_PARALLEL);
1146 }
1147
1148 static int
1149 test_multi_port_queue_parallel_to_atomic(void)
1150 {
1151         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
1152                                 RTE_SCHED_TYPE_ATOMIC);
1153 }
1154
1155 static int
1156 test_multi_port_queue_parallel_to_ordered(void)
1157 {
1158         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
1159                                 RTE_SCHED_TYPE_ORDERED);
1160 }
1161
1162 static int
1163 test_multi_port_queue_parallel_to_parallel(void)
1164 {
1165         return test_multiport_queue_sched_type_test(RTE_SCHED_TYPE_PARALLEL,
1166                                 RTE_SCHED_TYPE_PARALLEL);
1167 }
1168
1169 static int
1170 worker_flow_based_pipeline_max_stages_rand_sched_type(void *arg)
1171 {
1172         struct test_core_param *param = arg;
1173         struct rte_event ev;
1174         uint16_t valid_event;
1175         uint8_t port = param->port;
1176         rte_atomic32_t *total_events = param->total_events;
1177
1178         while (rte_atomic32_read(total_events) > 0) {
1179                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
1180                 if (!valid_event)
1181                         continue;
1182
1183                 if (ev.sub_event_type == 255) { /* last stage */
1184                         rte_pktmbuf_free(ev.mbuf);
1185                         rte_atomic32_sub(total_events, 1);
1186                 } else {
1187                         ev.event_type = RTE_EVENT_TYPE_CPU;
1188                         ev.sub_event_type++;
1189                         ev.sched_type =
1190                                 rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1);
1191                         ev.op = RTE_EVENT_OP_FORWARD;
1192                         rte_event_enqueue_burst(evdev, port, &ev, 1);
1193                 }
1194         }
1195         return 0;
1196 }
1197
1198 static int
1199 launch_multi_port_max_stages_random_sched_type(int (*fn)(void *))
1200 {
1201         uint32_t nr_ports;
1202         int ret;
1203
1204         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
1205                                 RTE_EVENT_DEV_ATTR_PORT_COUNT,
1206                                 &nr_ports), "Port count get failed");
1207         nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1);
1208
1209         if (!nr_ports) {
1210                 printf("%s: Not enough ports=%d or workers=%d\n", __func__,
1211                         nr_ports, rte_lcore_count() - 1);
1212                 return TEST_SUCCESS;
1213         }
1214
1215         /* Injects events with m->seqn=0 to total_events */
1216         ret = inject_events(
1217                 0x1 /*flow_id */,
1218                 RTE_EVENT_TYPE_CPU /* event_type */,
1219                 0 /* sub_event_type (stage 0) */,
1220                 rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1) /* sched_type */,
1221                 0 /* queue */,
1222                 0 /* port */,
1223                 MAX_EVENTS /* events */);
1224         if (ret)
1225                 return TEST_FAILED;
1226
1227         return launch_workers_and_wait(fn, fn, MAX_EVENTS, nr_ports,
1228                                          0xff /* invalid */);
1229 }
1230
1231 /* Flow based pipeline with maximum stages with random sched type */
1232 static int
1233 test_multi_port_flow_max_stages_random_sched_type(void)
1234 {
1235         return launch_multi_port_max_stages_random_sched_type(
1236                 worker_flow_based_pipeline_max_stages_rand_sched_type);
1237 }
1238
1239 static int
1240 worker_queue_based_pipeline_max_stages_rand_sched_type(void *arg)
1241 {
1242         struct test_core_param *param = arg;
1243         struct rte_event ev;
1244         uint16_t valid_event;
1245         uint8_t port = param->port;
1246         uint32_t queue_count;
1247         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
1248                             RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
1249                             &queue_count), "Queue count get failed");
1250         uint8_t nr_queues = queue_count;
1251         rte_atomic32_t *total_events = param->total_events;
1252
1253         while (rte_atomic32_read(total_events) > 0) {
1254                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
1255                 if (!valid_event)
1256                         continue;
1257
1258                 if (ev.queue_id == nr_queues - 1) { /* last stage */
1259                         rte_pktmbuf_free(ev.mbuf);
1260                         rte_atomic32_sub(total_events, 1);
1261                 } else {
1262                         ev.event_type = RTE_EVENT_TYPE_CPU;
1263                         ev.queue_id++;
1264                         ev.sched_type =
1265                                 rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1);
1266                         ev.op = RTE_EVENT_OP_FORWARD;
1267                         rte_event_enqueue_burst(evdev, port, &ev, 1);
1268                 }
1269         }
1270         return 0;
1271 }
1272
1273 /* Queue based pipeline with maximum stages with random sched type */
1274 static int
1275 test_multi_port_queue_max_stages_random_sched_type(void)
1276 {
1277         return launch_multi_port_max_stages_random_sched_type(
1278                 worker_queue_based_pipeline_max_stages_rand_sched_type);
1279 }
1280
1281 static int
1282 worker_mixed_pipeline_max_stages_rand_sched_type(void *arg)
1283 {
1284         struct test_core_param *param = arg;
1285         struct rte_event ev;
1286         uint16_t valid_event;
1287         uint8_t port = param->port;
1288         uint32_t queue_count;
1289         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
1290                             RTE_EVENT_DEV_ATTR_QUEUE_COUNT,
1291                             &queue_count), "Queue count get failed");
1292         uint8_t nr_queues = queue_count;
1293         rte_atomic32_t *total_events = param->total_events;
1294
1295         while (rte_atomic32_read(total_events) > 0) {
1296                 valid_event = rte_event_dequeue_burst(evdev, port, &ev, 1, 0);
1297                 if (!valid_event)
1298                         continue;
1299
1300                 if (ev.queue_id == nr_queues - 1) { /* Last stage */
1301                         rte_pktmbuf_free(ev.mbuf);
1302                         rte_atomic32_sub(total_events, 1);
1303                 } else {
1304                         ev.event_type = RTE_EVENT_TYPE_CPU;
1305                         ev.queue_id++;
1306                         ev.sub_event_type = rte_rand() % 256;
1307                         ev.sched_type =
1308                                 rte_rand() % (RTE_SCHED_TYPE_PARALLEL + 1);
1309                         ev.op = RTE_EVENT_OP_FORWARD;
1310                         rte_event_enqueue_burst(evdev, port, &ev, 1);
1311                 }
1312         }
1313         return 0;
1314 }
1315
1316 /* Queue and flow based pipeline with maximum stages with random sched type */
1317 static int
1318 test_multi_port_mixed_max_stages_random_sched_type(void)
1319 {
1320         return launch_multi_port_max_stages_random_sched_type(
1321                 worker_mixed_pipeline_max_stages_rand_sched_type);
1322 }
1323
1324 static int
1325 worker_ordered_flow_producer(void *arg)
1326 {
1327         struct test_core_param *param = arg;
1328         uint8_t port = param->port;
1329         struct rte_mbuf *m;
1330         int counter = 0;
1331
1332         while (counter < NUM_PACKETS) {
1333                 m = rte_pktmbuf_alloc(eventdev_test_mempool);
1334                 if (m == NULL)
1335                         continue;
1336
1337                 m->seqn = counter++;
1338
1339                 struct rte_event ev = {.event = 0, .u64 = 0};
1340
1341                 ev.flow_id = 0x1; /* Generate a fat flow */
1342                 ev.sub_event_type = 0;
1343                 /* Inject the new event */
1344                 ev.op = RTE_EVENT_OP_NEW;
1345                 ev.event_type = RTE_EVENT_TYPE_CPU;
1346                 ev.sched_type = RTE_SCHED_TYPE_ORDERED;
1347                 ev.queue_id = 0;
1348                 ev.mbuf = m;
1349                 rte_event_enqueue_burst(evdev, port, &ev, 1);
1350         }
1351
1352         return 0;
1353 }
1354
1355 static inline int
1356 test_producer_consumer_ingress_order_test(int (*fn)(void *))
1357 {
1358         uint32_t nr_ports;
1359
1360         TEST_ASSERT_SUCCESS(rte_event_dev_attr_get(evdev,
1361                                 RTE_EVENT_DEV_ATTR_PORT_COUNT,
1362                                 &nr_ports), "Port count get failed");
1363         nr_ports = RTE_MIN(nr_ports, rte_lcore_count() - 1);
1364
1365         if (rte_lcore_count() < 3 || nr_ports < 2) {
1366                 printf("### Not enough cores for %s test.\n", __func__);
1367                 return TEST_SUCCESS;
1368         }
1369
1370         launch_workers_and_wait(worker_ordered_flow_producer, fn,
1371                                 NUM_PACKETS, nr_ports, RTE_SCHED_TYPE_ATOMIC);
1372         /* Check the events order maintained or not */
1373         return seqn_list_check(NUM_PACKETS);
1374 }
1375
1376 /* Flow based producer consumer ingress order test */
1377 static int
1378 test_flow_producer_consumer_ingress_order_test(void)
1379 {
1380         return test_producer_consumer_ingress_order_test(
1381                                 worker_flow_based_pipeline);
1382 }
1383
1384 /* Queue based producer consumer ingress order test */
1385 static int
1386 test_queue_producer_consumer_ingress_order_test(void)
1387 {
1388         return test_producer_consumer_ingress_order_test(
1389                                 worker_group_based_pipeline);
1390 }
1391
1392 static struct unit_test_suite eventdev_octeontx_testsuite  = {
1393         .suite_name = "eventdev octeontx unit test suite",
1394         .setup = testsuite_setup,
1395         .teardown = testsuite_teardown,
1396         .unit_test_cases = {
1397                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1398                         test_simple_enqdeq_ordered),
1399                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1400                         test_simple_enqdeq_atomic),
1401                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1402                         test_simple_enqdeq_parallel),
1403                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1404                         test_multi_queue_enq_single_port_deq),
1405                 TEST_CASE_ST(eventdev_setup_priority, eventdev_teardown,
1406                         test_multi_queue_priority),
1407                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1408                         test_multi_queue_enq_multi_port_deq),
1409                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1410                         test_queue_to_port_single_link),
1411                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1412                         test_queue_to_port_multi_link),
1413                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1414                         test_multi_port_flow_ordered_to_atomic),
1415                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1416                         test_multi_port_flow_ordered_to_ordered),
1417                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1418                         test_multi_port_flow_ordered_to_parallel),
1419                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1420                         test_multi_port_flow_atomic_to_atomic),
1421                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1422                         test_multi_port_flow_atomic_to_ordered),
1423                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1424                         test_multi_port_flow_atomic_to_parallel),
1425                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1426                         test_multi_port_flow_parallel_to_atomic),
1427                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1428                         test_multi_port_flow_parallel_to_ordered),
1429                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1430                         test_multi_port_flow_parallel_to_parallel),
1431                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1432                         test_multi_port_queue_ordered_to_atomic),
1433                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1434                         test_multi_port_queue_ordered_to_ordered),
1435                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1436                         test_multi_port_queue_ordered_to_parallel),
1437                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1438                         test_multi_port_queue_atomic_to_atomic),
1439                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1440                         test_multi_port_queue_atomic_to_ordered),
1441                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1442                         test_multi_port_queue_atomic_to_parallel),
1443                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1444                         test_multi_port_queue_parallel_to_atomic),
1445                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1446                         test_multi_port_queue_parallel_to_ordered),
1447                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1448                         test_multi_port_queue_parallel_to_parallel),
1449                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1450                         test_multi_port_flow_max_stages_random_sched_type),
1451                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1452                         test_multi_port_queue_max_stages_random_sched_type),
1453                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1454                         test_multi_port_mixed_max_stages_random_sched_type),
1455                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1456                         test_flow_producer_consumer_ingress_order_test),
1457                 TEST_CASE_ST(eventdev_setup, eventdev_teardown,
1458                         test_queue_producer_consumer_ingress_order_test),
1459                 /* Tests with dequeue timeout */
1460                 TEST_CASE_ST(eventdev_setup_dequeue_timeout, eventdev_teardown,
1461                         test_multi_port_flow_ordered_to_atomic),
1462                 TEST_CASE_ST(eventdev_setup_dequeue_timeout, eventdev_teardown,
1463                         test_multi_port_queue_ordered_to_atomic),
1464                 TEST_CASES_END() /**< NULL terminate unit test array */
1465         }
1466 };
1467
1468 static int
1469 test_eventdev_octeontx(void)
1470 {
1471         return unit_test_suite_runner(&eventdev_octeontx_testsuite);
1472 }
1473
1474 REGISTER_TEST_COMMAND(eventdev_octeontx_autotest, test_eventdev_octeontx);