app/test-eventdev/test_pipeline_queue.c

   1 /*
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  * Copyright 2017 Cavium, Inc.
   4  */
   5
   6 #include "test_pipeline_common.h"
   7
   8 /* See http://dpdk.org/doc/guides/tools/testeventdev.html for test details */
   9
  10 static __rte_always_inline int
  11 pipeline_queue_nb_event_queues(struct evt_options *opt)
  12 {
  13         uint16_t eth_count = rte_eth_dev_count_avail();
  14
  15         return (eth_count * opt->nb_stages) + eth_count;
  16 }
  17
  18 static __rte_noinline int
  19 pipeline_queue_worker_single_stage_tx(void *arg)
  20 {
  21         PIPELINE_WORKER_SINGLE_STAGE_INIT;
  22
  23         while (t->done == false) {
  24                 uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
  25
  26                 if (!event) {
  27                         rte_pause();
  28                         continue;
  29                 }
  30
  31                 if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
  32                         pipeline_event_tx(dev, port, &ev);
  33                         w->processed_pkts++;
  34                 } else {
  35                         ev.queue_id++;
  36                         pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
  37                         pipeline_event_enqueue(dev, port, &ev);
  38                 }
  39         }
  40
  41         return 0;
  42 }
  43
  44 static __rte_noinline int
  45 pipeline_queue_worker_single_stage_fwd(void *arg)
  46 {
  47         PIPELINE_WORKER_SINGLE_STAGE_INIT;
  48         const uint8_t *tx_queue = t->tx_evqueue_id;
  49
  50         while (t->done == false) {
  51                 uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
  52
  53                 if (!event) {
  54                         rte_pause();
  55                         continue;
  56                 }
  57
  58                 ev.queue_id = tx_queue[ev.mbuf->port];
  59                 rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0);
  60                 pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
  61                 pipeline_event_enqueue(dev, port, &ev);
  62                 w->processed_pkts++;
  63         }
  64
  65         return 0;
  66 }
  67
  68 static __rte_noinline int
  69 pipeline_queue_worker_single_stage_burst_tx(void *arg)
  70 {
  71         PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT;
  72
  73         while (t->done == false) {
  74                 uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
  75                                 BURST_SIZE, 0);
  76
  77                 if (!nb_rx) {
  78                         rte_pause();
  79                         continue;
  80                 }
  81
  82                 for (i = 0; i < nb_rx; i++) {
  83                         rte_prefetch0(ev[i + 1].mbuf);
  84                         if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
  85                                 pipeline_event_tx(dev, port, &ev[i]);
  86                                 ev[i].op = RTE_EVENT_OP_RELEASE;
  87                                 w->processed_pkts++;
  88                         } else {
  89                                 ev[i].queue_id++;
  90                                 pipeline_fwd_event(&ev[i],
  91                                                 RTE_SCHED_TYPE_ATOMIC);
  92                         }
  93                 }
  94
  95                 pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
  96         }
  97
  98         return 0;
  99 }
 100
 101 static __rte_noinline int
 102 pipeline_queue_worker_single_stage_burst_fwd(void *arg)
 103 {
 104         PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT;
 105         const uint8_t *tx_queue = t->tx_evqueue_id;
 106
 107         while (t->done == false) {
 108                 uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
 109                                 BURST_SIZE, 0);
 110
 111                 if (!nb_rx) {
 112                         rte_pause();
 113                         continue;
 114                 }
 115
 116                 for (i = 0; i < nb_rx; i++) {
 117                         rte_prefetch0(ev[i + 1].mbuf);
 118                         ev[i].queue_id = tx_queue[ev[i].mbuf->port];
 119                         rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0);
 120                         pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC);
 121                 }
 122
 123                 pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
 124                 w->processed_pkts += nb_rx;
 125         }
 126
 127         return 0;
 128 }
 129
 130
 131 static __rte_noinline int
 132 pipeline_queue_worker_multi_stage_tx(void *arg)
 133 {
 134         PIPELINE_WORKER_MULTI_STAGE_INIT;
 135         const uint8_t *tx_queue = t->tx_evqueue_id;
 136
 137         while (t->done == false) {
 138                 uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
 139
 140                 if (!event) {
 141                         rte_pause();
 142                         continue;
 143                 }
 144
 145                 cq_id = ev.queue_id % nb_stages;
 146
 147                 if (ev.queue_id == tx_queue[ev.mbuf->port]) {
 148                         pipeline_event_tx(dev, port, &ev);
 149                         w->processed_pkts++;
 150                         continue;
 151                 }
 152
 153                 ev.queue_id++;
 154                 pipeline_fwd_event(&ev, cq_id != last_queue ?
 155                                 sched_type_list[cq_id] :
 156                                 RTE_SCHED_TYPE_ATOMIC);
 157                 pipeline_event_enqueue(dev, port, &ev);
 158         }
 159
 160         return 0;
 161 }
 162
 163 static __rte_noinline int
 164 pipeline_queue_worker_multi_stage_fwd(void *arg)
 165 {
 166         PIPELINE_WORKER_MULTI_STAGE_INIT;
 167         const uint8_t *tx_queue = t->tx_evqueue_id;
 168
 169         while (t->done == false) {
 170                 uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
 171
 172                 if (!event) {
 173                         rte_pause();
 174                         continue;
 175                 }
 176
 177                 cq_id = ev.queue_id % nb_stages;
 178
 179                 if (cq_id == last_queue) {
 180                         ev.queue_id = tx_queue[ev.mbuf->port];
 181                         rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0);
 182                         pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
 183                         w->processed_pkts++;
 184                 } else {
 185                         ev.queue_id++;
 186                         pipeline_fwd_event(&ev, sched_type_list[cq_id]);
 187                 }
 188
 189                 pipeline_event_enqueue(dev, port, &ev);
 190         }
 191
 192         return 0;
 193 }
 194
 195 static __rte_noinline int
 196 pipeline_queue_worker_multi_stage_burst_tx(void *arg)
 197 {
 198         PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
 199         const uint8_t *tx_queue = t->tx_evqueue_id;
 200
 201         while (t->done == false) {
 202                 uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
 203                                 BURST_SIZE, 0);
 204
 205                 if (!nb_rx) {
 206                         rte_pause();
 207                         continue;
 208                 }
 209
 210                 for (i = 0; i < nb_rx; i++) {
 211                         rte_prefetch0(ev[i + 1].mbuf);
 212                         cq_id = ev[i].queue_id % nb_stages;
 213
 214                         if (ev[i].queue_id == tx_queue[ev[i].mbuf->port]) {
 215                                 pipeline_event_tx(dev, port, &ev[i]);
 216                                 ev[i].op = RTE_EVENT_OP_RELEASE;
 217                                 w->processed_pkts++;
 218                                 continue;
 219                         }
 220
 221                         ev[i].queue_id++;
 222                         pipeline_fwd_event(&ev[i], cq_id != last_queue ?
 223                                         sched_type_list[cq_id] :
 224                                         RTE_SCHED_TYPE_ATOMIC);
 225                 }
 226
 227                 pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
 228         }
 229
 230         return 0;
 231 }
 232
 233 static __rte_noinline int
 234 pipeline_queue_worker_multi_stage_burst_fwd(void *arg)
 235 {
 236         PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
 237         const uint8_t *tx_queue = t->tx_evqueue_id;
 238
 239         while (t->done == false) {
 240                 uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev,
 241                                 BURST_SIZE, 0);
 242
 243                 if (!nb_rx) {
 244                         rte_pause();
 245                         continue;
 246                 }
 247
 248                 for (i = 0; i < nb_rx; i++) {
 249                         rte_prefetch0(ev[i + 1].mbuf);
 250                         cq_id = ev[i].queue_id % nb_stages;
 251
 252                         if (cq_id == last_queue) {
 253                                 ev[i].queue_id = tx_queue[ev[i].mbuf->port];
 254                                 rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0);
 255                                 pipeline_fwd_event(&ev[i],
 256                                                 RTE_SCHED_TYPE_ATOMIC);
 257                                 w->processed_pkts++;
 258                         } else {
 259                                 ev[i].queue_id++;
 260                                 pipeline_fwd_event(&ev[i],
 261                                                 sched_type_list[cq_id]);
 262                         }
 263                 }
 264
 265                 pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
 266         }
 267
 268         return 0;
 269 }
 270
 271 static int
 272 worker_wrapper(void *arg)
 273 {
 274         struct worker_data *w  = arg;
 275         struct evt_options *opt = w->t->opt;
 276         const bool burst = evt_has_burst_mode(w->dev_id);
 277         const bool internal_port = w->t->internal_port;
 278         const uint8_t nb_stages = opt->nb_stages;
 279         RTE_SET_USED(opt);
 280
 281         if (nb_stages == 1) {
 282                 if (!burst && internal_port)
 283                         return pipeline_queue_worker_single_stage_tx(arg);
 284                 else if (!burst && !internal_port)
 285                         return pipeline_queue_worker_single_stage_fwd(arg);
 286                 else if (burst && internal_port)
 287                         return pipeline_queue_worker_single_stage_burst_tx(arg);
 288                 else if (burst && !internal_port)
 289                         return pipeline_queue_worker_single_stage_burst_fwd(
 290                                         arg);
 291         } else {
 292                 if (!burst && internal_port)
 293                         return pipeline_queue_worker_multi_stage_tx(arg);
 294                 else if (!burst && !internal_port)
 295                         return pipeline_queue_worker_multi_stage_fwd(arg);
 296                 else if (burst && internal_port)
 297                         return pipeline_queue_worker_multi_stage_burst_tx(arg);
 298                 else if (burst && !internal_port)
 299                         return pipeline_queue_worker_multi_stage_burst_fwd(arg);
 300
 301         }
 302         rte_panic("invalid worker\n");
 303 }
 304
 305 static int
 306 pipeline_queue_launch_lcores(struct evt_test *test, struct evt_options *opt)
 307 {
 308         return pipeline_launch_lcores(test, opt, worker_wrapper);
 309 }
 310
 311 static int
 312 pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
 313 {
 314         int ret;
 315         int nb_ports;
 316         int nb_queues;
 317         int nb_stages = opt->nb_stages;
 318         uint8_t queue;
 319         uint8_t tx_evport_id = 0;
 320         uint8_t tx_evqueue_id[RTE_MAX_ETHPORTS];
 321         uint8_t queue_arr[RTE_EVENT_MAX_QUEUES_PER_DEV];
 322         uint8_t nb_worker_queues = 0;
 323         uint16_t prod = 0;
 324         struct rte_event_dev_info info;
 325         struct test_pipeline *t = evt_test_priv(test);
 326
 327         nb_ports = evt_nr_active_lcores(opt->wlcores);
 328         nb_queues = rte_eth_dev_count_avail() * (nb_stages);
 329
 330         /* One queue for Tx adapter per port */
 331         nb_queues += rte_eth_dev_count_avail();
 332
 333         memset(tx_evqueue_id, 0, sizeof(uint8_t) * RTE_MAX_ETHPORTS);
 334         memset(queue_arr, 0, sizeof(uint8_t) * RTE_EVENT_MAX_QUEUES_PER_DEV);
 335
 336         rte_event_dev_info_get(opt->dev_id, &info);
 337         const struct rte_event_dev_config config = {
 338                         .nb_event_queues = nb_queues,
 339                         .nb_event_ports = nb_ports,
 340                         .nb_events_limit  = info.max_num_events,
 341                         .nb_event_queue_flows = opt->nb_flows,
 342                         .nb_event_port_dequeue_depth =
 343                                 info.max_event_port_dequeue_depth,
 344                         .nb_event_port_enqueue_depth =
 345                                 info.max_event_port_enqueue_depth,
 346         };
 347         ret = rte_event_dev_configure(opt->dev_id, &config);
 348         if (ret) {
 349                 evt_err("failed to configure eventdev %d", opt->dev_id);
 350                 return ret;
 351         }
 352
 353         struct rte_event_queue_conf q_conf = {
 354                         .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 355                         .nb_atomic_flows = opt->nb_flows,
 356                         .nb_atomic_order_sequences = opt->nb_flows,
 357         };
 358         /* queue configurations */
 359         for (queue = 0; queue < nb_queues; queue++) {
 360                 uint8_t slot;
 361
 362                 q_conf.event_queue_cfg = 0;
 363                 slot = queue % (nb_stages + 1);
 364                 if (slot == nb_stages) {
 365                         q_conf.schedule_type = RTE_SCHED_TYPE_ATOMIC;
 366                         if (!t->internal_port) {
 367                                 q_conf.event_queue_cfg =
 368                                         RTE_EVENT_QUEUE_CFG_SINGLE_LINK;
 369                         }
 370                         tx_evqueue_id[prod++] = queue;
 371                 } else {
 372                         q_conf.schedule_type = opt->sched_type_list[slot];
 373                         queue_arr[nb_worker_queues] = queue;
 374                         nb_worker_queues++;
 375                 }
 376
 377                 ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf);
 378                 if (ret) {
 379                         evt_err("failed to setup queue=%d", queue);
 380                         return ret;
 381                 }
 382         }
 383
 384         if (opt->wkr_deq_dep > info.max_event_port_dequeue_depth)
 385                 opt->wkr_deq_dep = info.max_event_port_dequeue_depth;
 386
 387         /* port configuration */
 388         const struct rte_event_port_conf p_conf = {
 389                         .dequeue_depth = opt->wkr_deq_dep,
 390                         .enqueue_depth = info.max_event_port_dequeue_depth,
 391                         .new_event_threshold = info.max_num_events,
 392         };
 393
 394         if (!t->internal_port) {
 395                 ret = pipeline_event_port_setup(test, opt, queue_arr,
 396                                 nb_worker_queues, p_conf);
 397                 if (ret)
 398                         return ret;
 399         } else
 400                 ret = pipeline_event_port_setup(test, opt, NULL, nb_queues,
 401                                 p_conf);
 402
 403         if (ret)
 404                 return ret;
 405         /*
 406          * The pipelines are setup in the following manner:
 407          *
 408          * eth_dev_count = 2, nb_stages = 2.
 409          *
 410          *      queues = 6
 411          *      stride = 3
 412          *
 413          *      event queue pipelines:
 414          *      eth0 -> q0 -> q1 -> (q2->tx)
 415          *      eth1 -> q3 -> q4 -> (q5->tx)
 416          *
 417          *      q2, q5 configured as ATOMIC | SINGLE_LINK
 418          *
 419          */
 420         ret = pipeline_event_rx_adapter_setup(opt, nb_stages + 1, p_conf);
 421         if (ret)
 422                 return ret;
 423
 424         ret = pipeline_event_tx_adapter_setup(opt, p_conf);
 425         if (ret)
 426                 return ret;
 427
 428         if (!evt_has_distributed_sched(opt->dev_id)) {
 429                 uint32_t service_id;
 430                 rte_event_dev_service_id_get(opt->dev_id, &service_id);
 431                 ret = evt_service_setup(service_id);
 432                 if (ret) {
 433                         evt_err("No service lcore found to run event dev.");
 434                         return ret;
 435                 }
 436         }
 437
 438         /* Connect the tx_evqueue_id to the Tx adapter port */
 439         if (!t->internal_port) {
 440                 RTE_ETH_FOREACH_DEV(prod) {
 441                         ret = rte_event_eth_tx_adapter_event_port_get(prod,
 442                                         &tx_evport_id);
 443                         if (ret) {
 444                                 evt_err("Unable to get Tx adptr[%d] evprt[%d]",
 445                                                 prod, tx_evport_id);
 446                                 return ret;
 447                         }
 448
 449                         if (rte_event_port_link(opt->dev_id, tx_evport_id,
 450                                                 &tx_evqueue_id[prod],
 451                                                 NULL, 1) != 1) {
 452                                 evt_err("Unable to link Tx adptr[%d] evprt[%d]",
 453                                                 prod, tx_evport_id);
 454                                 return ret;
 455                         }
 456                 }
 457         }
 458
 459         RTE_ETH_FOREACH_DEV(prod) {
 460                 ret = rte_eth_dev_start(prod);
 461                 if (ret) {
 462                         evt_err("Ethernet dev [%d] failed to start."
 463                                         " Using synthetic producer", prod);
 464                         return ret;
 465                 }
 466
 467         }
 468
 469         ret = rte_event_dev_start(opt->dev_id);
 470         if (ret) {
 471                 evt_err("failed to start eventdev %d", opt->dev_id);
 472                 return ret;
 473         }
 474
 475         RTE_ETH_FOREACH_DEV(prod) {
 476                 ret = rte_event_eth_rx_adapter_start(prod);
 477                 if (ret) {
 478                         evt_err("Rx adapter[%d] start failed", prod);
 479                         return ret;
 480                 }
 481
 482                 ret = rte_event_eth_tx_adapter_start(prod);
 483                 if (ret) {
 484                         evt_err("Tx adapter[%d] start failed", prod);
 485                         return ret;
 486                 }
 487         }
 488
 489         memcpy(t->tx_evqueue_id, tx_evqueue_id, sizeof(uint8_t) *
 490                         RTE_MAX_ETHPORTS);
 491
 492         return 0;
 493 }
 494
 495 static void
 496 pipeline_queue_opt_dump(struct evt_options *opt)
 497 {
 498         pipeline_opt_dump(opt, pipeline_queue_nb_event_queues(opt));
 499 }
 500
 501 static int
 502 pipeline_queue_opt_check(struct evt_options *opt)
 503 {
 504         return pipeline_opt_check(opt, pipeline_queue_nb_event_queues(opt));
 505 }
 506
 507 static bool
 508 pipeline_queue_capability_check(struct evt_options *opt)
 509 {
 510         struct rte_event_dev_info dev_info;
 511
 512         rte_event_dev_info_get(opt->dev_id, &dev_info);
 513         if (dev_info.max_event_queues < pipeline_queue_nb_event_queues(opt) ||
 514                         dev_info.max_event_ports <
 515                         evt_nr_active_lcores(opt->wlcores)) {
 516                 evt_err("not enough eventdev queues=%d/%d or ports=%d/%d",
 517                         pipeline_queue_nb_event_queues(opt),
 518                         dev_info.max_event_queues,
 519                         evt_nr_active_lcores(opt->wlcores),
 520                         dev_info.max_event_ports);
 521         }
 522
 523         return true;
 524 }
 525
 526 static const struct evt_test_ops pipeline_queue =  {
 527         .cap_check          = pipeline_queue_capability_check,
 528         .opt_check          = pipeline_queue_opt_check,
 529         .opt_dump           = pipeline_queue_opt_dump,
 530         .test_setup         = pipeline_test_setup,
 531         .mempool_setup      = pipeline_mempool_setup,
 532         .ethdev_setup       = pipeline_ethdev_setup,
 533         .eventdev_setup     = pipeline_queue_eventdev_setup,
 534         .launch_lcores      = pipeline_queue_launch_lcores,
 535         .eventdev_destroy   = pipeline_eventdev_destroy,
 536         .mempool_destroy    = pipeline_mempool_destroy,
 537         .ethdev_destroy     = pipeline_ethdev_destroy,
 538         .test_result        = pipeline_test_result,
 539         .test_destroy       = pipeline_test_destroy,
 540 };
 541
 542 EVT_TEST_REGISTER(pipeline_queue);