app/test-eventdev/test_perf_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Cavium, Inc
   3  */
   4
   5 #include "test_perf_common.h"
   6
   7 int
   8 perf_test_result(struct evt_test *test, struct evt_options *opt)
   9 {
  10         RTE_SET_USED(opt);
  11         int i;
  12         uint64_t total = 0;
  13         struct test_perf *t = evt_test_priv(test);
  14
  15         printf("Packet distribution across worker cores :\n");
  16         for (i = 0; i < t->nb_workers; i++)
  17                 total += t->worker[i].processed_pkts;
  18         for (i = 0; i < t->nb_workers; i++)
  19                 printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:"
  20                                 CLGRN" %3.2f\n"CLNRM, i,
  21                                 t->worker[i].processed_pkts,
  22                                 (((double)t->worker[i].processed_pkts)/total)
  23                                 * 100);
  24
  25         return t->result;
  26 }
  27
  28 static inline int
  29 perf_producer(void *arg)
  30 {
  31         struct prod_data *p  = arg;
  32         struct test_perf *t = p->t;
  33         struct evt_options *opt = t->opt;
  34         const uint8_t dev_id = p->dev_id;
  35         const uint8_t port = p->port_id;
  36         struct rte_mempool *pool = t->pool;
  37         const uint64_t nb_pkts = t->nb_pkts;
  38         const uint32_t nb_flows = t->nb_flows;
  39         uint32_t flow_counter = 0;
  40         uint64_t count = 0;
  41         struct perf_elt *m;
  42         struct rte_event ev;
  43
  44         if (opt->verbose_level > 1)
  45                 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__,
  46                                 rte_lcore_id(), dev_id, port, p->queue_id);
  47
  48         ev.event = 0;
  49         ev.op = RTE_EVENT_OP_NEW;
  50         ev.queue_id = p->queue_id;
  51         ev.sched_type = t->opt->sched_type_list[0];
  52         ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
  53         ev.event_type =  RTE_EVENT_TYPE_CPU;
  54         ev.sub_event_type = 0; /* stage 0 */
  55
  56         while (count < nb_pkts && t->done == false) {
  57                 if (rte_mempool_get(pool, (void **)&m) < 0)
  58                         continue;
  59
  60                 ev.flow_id = flow_counter++ % nb_flows;
  61                 ev.event_ptr = m;
  62                 m->timestamp = rte_get_timer_cycles();
  63                 while (rte_event_enqueue_burst(dev_id, port, &ev, 1) != 1) {
  64                         if (t->done)
  65                                 break;
  66                         rte_pause();
  67                         m->timestamp = rte_get_timer_cycles();
  68                 }
  69                 count++;
  70         }
  71
  72         return 0;
  73 }
  74
  75 static inline int
  76 perf_event_timer_producer(void *arg)
  77 {
  78         struct prod_data *p  = arg;
  79         struct test_perf *t = p->t;
  80         struct evt_options *opt = t->opt;
  81         uint32_t flow_counter = 0;
  82         uint64_t count = 0;
  83         uint64_t arm_latency = 0;
  84         const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs;
  85         const uint32_t nb_flows = t->nb_flows;
  86         const uint64_t nb_timers = opt->nb_timers;
  87         struct rte_mempool *pool = t->pool;
  88         struct perf_elt *m;
  89         struct rte_event_timer_adapter **adptr = t->timer_adptr;
  90         struct rte_event_timer tim;
  91         uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec;
  92
  93         memset(&tim, 0, sizeof(struct rte_event_timer));
  94         timeout_ticks = opt->optm_timer_tick_nsec ?
  95                         (timeout_ticks * opt->timer_tick_nsec)
  96                         / opt->optm_timer_tick_nsec : timeout_ticks;
  97         timeout_ticks += timeout_ticks ? 0 : 1;
  98         tim.ev.event_type =  RTE_EVENT_TYPE_TIMER;
  99         tim.ev.op = RTE_EVENT_OP_NEW;
 100         tim.ev.sched_type = t->opt->sched_type_list[0];
 101         tim.ev.queue_id = p->queue_id;
 102         tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
 103         tim.state = RTE_EVENT_TIMER_NOT_ARMED;
 104         tim.timeout_ticks = timeout_ticks;
 105
 106         if (opt->verbose_level > 1)
 107                 printf("%s(): lcore %d\n", __func__, rte_lcore_id());
 108
 109         while (count < nb_timers && t->done == false) {
 110                 if (rte_mempool_get(pool, (void **)&m) < 0)
 111                         continue;
 112
 113                 m->tim = tim;
 114                 m->tim.ev.flow_id = flow_counter++ % nb_flows;
 115                 m->tim.ev.event_ptr = m;
 116                 m->timestamp = rte_get_timer_cycles();
 117                 while (rte_event_timer_arm_burst(
 118                                 adptr[flow_counter % nb_timer_adptrs],
 119                                 (struct rte_event_timer **)&m, 1) != 1) {
 120                         if (t->done)
 121                                 break;
 122                         rte_pause();
 123                         m->timestamp = rte_get_timer_cycles();
 124                 }
 125                 arm_latency += rte_get_timer_cycles() - m->timestamp;
 126                 count++;
 127         }
 128         fflush(stdout);
 129         rte_delay_ms(1000);
 130         printf("%s(): lcore %d Average event timer arm latency = %.3f us\n",
 131                         __func__, rte_lcore_id(), (float)(arm_latency / count) /
 132                         (rte_get_timer_hz() / 1000000));
 133         return 0;
 134 }
 135
 136 static inline int
 137 perf_event_timer_producer_burst(void *arg)
 138 {
 139         int i;
 140         struct prod_data *p  = arg;
 141         struct test_perf *t = p->t;
 142         struct evt_options *opt = t->opt;
 143         uint32_t flow_counter = 0;
 144         uint64_t count = 0;
 145         uint64_t arm_latency = 0;
 146         const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs;
 147         const uint32_t nb_flows = t->nb_flows;
 148         const uint64_t nb_timers = opt->nb_timers;
 149         struct rte_mempool *pool = t->pool;
 150         struct perf_elt *m[BURST_SIZE + 1] = {NULL};
 151         struct rte_event_timer_adapter **adptr = t->timer_adptr;
 152         struct rte_event_timer tim;
 153         uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec;
 154
 155         memset(&tim, 0, sizeof(struct rte_event_timer));
 156         timeout_ticks = opt->optm_timer_tick_nsec ?
 157                         (timeout_ticks * opt->timer_tick_nsec)
 158                         / opt->optm_timer_tick_nsec : timeout_ticks;
 159         timeout_ticks += timeout_ticks ? 0 : 1;
 160         tim.ev.event_type =  RTE_EVENT_TYPE_TIMER;
 161         tim.ev.op = RTE_EVENT_OP_NEW;
 162         tim.ev.sched_type = t->opt->sched_type_list[0];
 163         tim.ev.queue_id = p->queue_id;
 164         tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
 165         tim.state = RTE_EVENT_TIMER_NOT_ARMED;
 166         tim.timeout_ticks = timeout_ticks;
 167
 168         if (opt->verbose_level > 1)
 169                 printf("%s(): lcore %d\n", __func__, rte_lcore_id());
 170
 171         while (count < nb_timers && t->done == false) {
 172                 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0)
 173                         continue;
 174                 for (i = 0; i < BURST_SIZE; i++) {
 175                         rte_prefetch0(m[i + 1]);
 176                         m[i]->tim = tim;
 177                         m[i]->tim.ev.flow_id = flow_counter++ % nb_flows;
 178                         m[i]->tim.ev.event_ptr = m[i];
 179                         m[i]->timestamp = rte_get_timer_cycles();
 180                 }
 181                 rte_event_timer_arm_tmo_tick_burst(
 182                                 adptr[flow_counter % nb_timer_adptrs],
 183                                 (struct rte_event_timer **)m,
 184                                 tim.timeout_ticks,
 185                                 BURST_SIZE);
 186                 arm_latency += rte_get_timer_cycles() - m[i - 1]->timestamp;
 187                 count += BURST_SIZE;
 188         }
 189         fflush(stdout);
 190         rte_delay_ms(1000);
 191         printf("%s(): lcore %d Average event timer arm latency = %.3f us\n",
 192                         __func__, rte_lcore_id(), (float)(arm_latency / count) /
 193                         (rte_get_timer_hz() / 1000000));
 194         return 0;
 195 }
 196
 197 static int
 198 perf_producer_wrapper(void *arg)
 199 {
 200         struct prod_data *p  = arg;
 201         struct test_perf *t = p->t;
 202         /* Launch the producer function only in case of synthetic producer. */
 203         if (t->opt->prod_type == EVT_PROD_TYPE_SYNT)
 204                 return perf_producer(arg);
 205         else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR &&
 206                         !t->opt->timdev_use_burst)
 207                 return perf_event_timer_producer(arg);
 208         else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR &&
 209                         t->opt->timdev_use_burst)
 210                 return perf_event_timer_producer_burst(arg);
 211         return 0;
 212 }
 213
 214 static inline uint64_t
 215 processed_pkts(struct test_perf *t)
 216 {
 217         uint8_t i;
 218         uint64_t total = 0;
 219
 220         rte_smp_rmb();
 221         for (i = 0; i < t->nb_workers; i++)
 222                 total += t->worker[i].processed_pkts;
 223
 224         return total;
 225 }
 226
 227 static inline uint64_t
 228 total_latency(struct test_perf *t)
 229 {
 230         uint8_t i;
 231         uint64_t total = 0;
 232
 233         rte_smp_rmb();
 234         for (i = 0; i < t->nb_workers; i++)
 235                 total += t->worker[i].latency;
 236
 237         return total;
 238 }
 239
 240
 241 int
 242 perf_launch_lcores(struct evt_test *test, struct evt_options *opt,
 243                 int (*worker)(void *))
 244 {
 245         int ret, lcore_id;
 246         struct test_perf *t = evt_test_priv(test);
 247
 248         int port_idx = 0;
 249         /* launch workers */
 250         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 251                 if (!(opt->wlcores[lcore_id]))
 252                         continue;
 253
 254                 ret = rte_eal_remote_launch(worker,
 255                                  &t->worker[port_idx], lcore_id);
 256                 if (ret) {
 257                         evt_err("failed to launch worker %d", lcore_id);
 258                         return ret;
 259                 }
 260                 port_idx++;
 261         }
 262
 263         /* launch producers */
 264         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 265                 if (!(opt->plcores[lcore_id]))
 266                         continue;
 267
 268                 ret = rte_eal_remote_launch(perf_producer_wrapper,
 269                                 &t->prod[port_idx], lcore_id);
 270                 if (ret) {
 271                         evt_err("failed to launch perf_producer %d", lcore_id);
 272                         return ret;
 273                 }
 274                 port_idx++;
 275         }
 276
 277         const uint64_t total_pkts = t->outstand_pkts;
 278
 279         uint64_t dead_lock_cycles = rte_get_timer_cycles();
 280         int64_t dead_lock_remaining  =  total_pkts;
 281         const uint64_t dead_lock_sample = rte_get_timer_hz() * 5;
 282
 283         uint64_t perf_cycles = rte_get_timer_cycles();
 284         int64_t perf_remaining  = total_pkts;
 285         const uint64_t perf_sample = rte_get_timer_hz();
 286
 287         static float total_mpps;
 288         static uint64_t samples;
 289
 290         const uint64_t freq_mhz = rte_get_timer_hz() / 1000000;
 291         int64_t remaining = t->outstand_pkts - processed_pkts(t);
 292
 293         while (t->done == false) {
 294                 const uint64_t new_cycles = rte_get_timer_cycles();
 295
 296                 if ((new_cycles - perf_cycles) > perf_sample) {
 297                         const uint64_t latency = total_latency(t);
 298                         const uint64_t pkts = processed_pkts(t);
 299
 300                         remaining = t->outstand_pkts - pkts;
 301                         float mpps = (float)(perf_remaining-remaining)/1000000;
 302
 303                         perf_remaining = remaining;
 304                         perf_cycles = new_cycles;
 305                         total_mpps += mpps;
 306                         ++samples;
 307                         if (opt->fwd_latency && pkts > 0) {
 308                                 printf(CLGRN"\r%.3f mpps avg %.3f mpps [avg fwd latency %.3f us] "CLNRM,
 309                                         mpps, total_mpps/samples,
 310                                         (float)(latency/pkts)/freq_mhz);
 311                         } else {
 312                                 printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM,
 313                                         mpps, total_mpps/samples);
 314                         }
 315                         fflush(stdout);
 316
 317                         if (remaining <= 0) {
 318                                 t->result = EVT_TEST_SUCCESS;
 319                                 if (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 320                                         opt->prod_type ==
 321                                         EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 322                                         t->done = true;
 323                                         rte_smp_wmb();
 324                                         break;
 325                                 }
 326                         }
 327                 }
 328
 329                 if (new_cycles - dead_lock_cycles > dead_lock_sample &&
 330                     (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 331                      opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR)) {
 332                         remaining = t->outstand_pkts - processed_pkts(t);
 333                         if (dead_lock_remaining == remaining) {
 334                                 rte_event_dev_dump(opt->dev_id, stdout);
 335                                 evt_err("No schedules for seconds, deadlock");
 336                                 t->done = true;
 337                                 rte_smp_wmb();
 338                                 break;
 339                         }
 340                         dead_lock_remaining = remaining;
 341                         dead_lock_cycles = new_cycles;
 342                 }
 343         }
 344         printf("\n");
 345         return 0;
 346 }
 347
 348 static int
 349 perf_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride,
 350                 struct rte_event_port_conf prod_conf)
 351 {
 352         int ret = 0;
 353         uint16_t prod;
 354         struct rte_event_eth_rx_adapter_queue_conf queue_conf;
 355
 356         memset(&queue_conf, 0,
 357                         sizeof(struct rte_event_eth_rx_adapter_queue_conf));
 358         queue_conf.ev.sched_type = opt->sched_type_list[0];
 359         RTE_ETH_FOREACH_DEV(prod) {
 360                 uint32_t cap;
 361
 362                 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id,
 363                                 prod, &cap);
 364                 if (ret) {
 365                         evt_err("failed to get event rx adapter[%d]"
 366                                         " capabilities",
 367                                         opt->dev_id);
 368                         return ret;
 369                 }
 370                 queue_conf.ev.queue_id = prod * stride;
 371                 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id,
 372                                 &prod_conf);
 373                 if (ret) {
 374                         evt_err("failed to create rx adapter[%d]", prod);
 375                         return ret;
 376                 }
 377                 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1,
 378                                 &queue_conf);
 379                 if (ret) {
 380                         evt_err("failed to add rx queues to adapter[%d]", prod);
 381                         return ret;
 382                 }
 383
 384                 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) {
 385                         uint32_t service_id;
 386
 387                         rte_event_eth_rx_adapter_service_id_get(prod,
 388                                         &service_id);
 389                         ret = evt_service_setup(service_id);
 390                         if (ret) {
 391                                 evt_err("Failed to setup service core"
 392                                                 " for Rx adapter\n");
 393                                 return ret;
 394                         }
 395                 }
 396
 397                 ret = rte_eth_dev_start(prod);
 398                 if (ret) {
 399                         evt_err("Ethernet dev [%d] failed to start."
 400                                         " Using synthetic producer", prod);
 401                         return ret;
 402                 }
 403
 404                 ret = rte_event_eth_rx_adapter_start(prod);
 405                 if (ret) {
 406                         evt_err("Rx adapter[%d] start failed", prod);
 407                         return ret;
 408                 }
 409                 printf("%s: Port[%d] using Rx adapter[%d] started\n", __func__,
 410                                 prod, prod);
 411         }
 412
 413         return ret;
 414 }
 415
 416 static int
 417 perf_event_timer_adapter_setup(struct test_perf *t)
 418 {
 419         int i;
 420         int ret;
 421         struct rte_event_timer_adapter_info adapter_info;
 422         struct rte_event_timer_adapter *wl;
 423         uint8_t nb_producers = evt_nr_active_lcores(t->opt->plcores);
 424         uint8_t flags = RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES;
 425
 426         if (nb_producers == 1)
 427                 flags |= RTE_EVENT_TIMER_ADAPTER_F_SP_PUT;
 428
 429         for (i = 0; i < t->opt->nb_timer_adptrs; i++) {
 430                 struct rte_event_timer_adapter_conf config = {
 431                         .event_dev_id = t->opt->dev_id,
 432                         .timer_adapter_id = i,
 433                         .timer_tick_ns = t->opt->timer_tick_nsec,
 434                         .max_tmo_ns = t->opt->max_tmo_nsec,
 435                         .nb_timers = 2 * 1024 * 1024,
 436                         .flags = flags,
 437                 };
 438
 439                 wl = rte_event_timer_adapter_create(&config);
 440                 if (wl == NULL) {
 441                         evt_err("failed to create event timer ring %d", i);
 442                         return rte_errno;
 443                 }
 444
 445                 memset(&adapter_info, 0,
 446                                 sizeof(struct rte_event_timer_adapter_info));
 447                 rte_event_timer_adapter_get_info(wl, &adapter_info);
 448                 t->opt->optm_timer_tick_nsec = adapter_info.min_resolution_ns;
 449
 450                 if (!(adapter_info.caps &
 451                                 RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) {
 452                         uint32_t service_id;
 453
 454                         rte_event_timer_adapter_service_id_get(wl,
 455                                         &service_id);
 456                         ret = evt_service_setup(service_id);
 457                         if (ret) {
 458                                 evt_err("Failed to setup service core"
 459                                                 " for timer adapter\n");
 460                                 return ret;
 461                         }
 462                         rte_service_runstate_set(service_id, 1);
 463                 }
 464
 465                 ret = rte_event_timer_adapter_start(wl);
 466                 if (ret) {
 467                         evt_err("failed to Start event timer adapter %d", i);
 468                         return ret;
 469                 }
 470                 t->timer_adptr[i] = wl;
 471         }
 472         return 0;
 473 }
 474
 475 int
 476 perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
 477                                 uint8_t stride, uint8_t nb_queues,
 478                                 const struct rte_event_port_conf *port_conf)
 479 {
 480         struct test_perf *t = evt_test_priv(test);
 481         uint16_t port, prod;
 482         int ret = -1;
 483
 484         /* setup one port per worker, linking to all queues */
 485         for (port = 0; port < evt_nr_active_lcores(opt->wlcores);
 486                                 port++) {
 487                 struct worker_data *w = &t->worker[port];
 488
 489                 w->dev_id = opt->dev_id;
 490                 w->port_id = port;
 491                 w->t = t;
 492                 w->processed_pkts = 0;
 493                 w->latency = 0;
 494
 495                 ret = rte_event_port_setup(opt->dev_id, port, port_conf);
 496                 if (ret) {
 497                         evt_err("failed to setup port %d", port);
 498                         return ret;
 499                 }
 500
 501                 ret = rte_event_port_link(opt->dev_id, port, NULL, NULL, 0);
 502                 if (ret != nb_queues) {
 503                         evt_err("failed to link all queues to port %d", port);
 504                         return -EINVAL;
 505                 }
 506         }
 507
 508         /* port for producers, no links */
 509         if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) {
 510                 for ( ; port < perf_nb_event_ports(opt); port++) {
 511                         struct prod_data *p = &t->prod[port];
 512                         p->t = t;
 513                 }
 514
 515                 ret = perf_event_rx_adapter_setup(opt, stride, *port_conf);
 516                 if (ret)
 517                         return ret;
 518         } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 519                 prod = 0;
 520                 for ( ; port < perf_nb_event_ports(opt); port++) {
 521                         struct prod_data *p = &t->prod[port];
 522                         p->queue_id = prod * stride;
 523                         p->t = t;
 524                         prod++;
 525                 }
 526
 527                 ret = perf_event_timer_adapter_setup(t);
 528                 if (ret)
 529                         return ret;
 530         } else {
 531                 prod = 0;
 532                 for ( ; port < perf_nb_event_ports(opt); port++) {
 533                         struct prod_data *p = &t->prod[port];
 534
 535                         p->dev_id = opt->dev_id;
 536                         p->port_id = port;
 537                         p->queue_id = prod * stride;
 538                         p->t = t;
 539
 540                         ret = rte_event_port_setup(opt->dev_id, port,
 541                                         port_conf);
 542                         if (ret) {
 543                                 evt_err("failed to setup port %d", port);
 544                                 return ret;
 545                         }
 546                         prod++;
 547                 }
 548         }
 549
 550         return ret;
 551 }
 552
 553 int
 554 perf_opt_check(struct evt_options *opt, uint64_t nb_queues)
 555 {
 556         unsigned int lcores;
 557
 558         /* N producer + N worker + 1 master when producer cores are used
 559          * Else N worker + 1 master when Rx adapter is used
 560          */
 561         lcores = opt->prod_type == EVT_PROD_TYPE_SYNT ? 3 : 2;
 562
 563         if (rte_lcore_count() < lcores) {
 564                 evt_err("test need minimum %d lcores", lcores);
 565                 return -1;
 566         }
 567
 568         /* Validate worker lcores */
 569         if (evt_lcores_has_overlap(opt->wlcores, rte_get_master_lcore())) {
 570                 evt_err("worker lcores overlaps with master lcore");
 571                 return -1;
 572         }
 573         if (evt_lcores_has_overlap_multi(opt->wlcores, opt->plcores)) {
 574                 evt_err("worker lcores overlaps producer lcores");
 575                 return -1;
 576         }
 577         if (evt_has_disabled_lcore(opt->wlcores)) {
 578                 evt_err("one or more workers lcores are not enabled");
 579                 return -1;
 580         }
 581         if (!evt_has_active_lcore(opt->wlcores)) {
 582                 evt_err("minimum one worker is required");
 583                 return -1;
 584         }
 585
 586         if (opt->prod_type == EVT_PROD_TYPE_SYNT) {
 587                 /* Validate producer lcores */
 588                 if (evt_lcores_has_overlap(opt->plcores,
 589                                         rte_get_master_lcore())) {
 590                         evt_err("producer lcores overlaps with master lcore");
 591                         return -1;
 592                 }
 593                 if (evt_has_disabled_lcore(opt->plcores)) {
 594                         evt_err("one or more producer lcores are not enabled");
 595                         return -1;
 596                 }
 597                 if (!evt_has_active_lcore(opt->plcores)) {
 598                         evt_err("minimum one producer is required");
 599                         return -1;
 600                 }
 601         }
 602
 603         if (evt_has_invalid_stage(opt))
 604                 return -1;
 605
 606         if (evt_has_invalid_sched_type(opt))
 607                 return -1;
 608
 609         if (nb_queues > EVT_MAX_QUEUES) {
 610                 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES);
 611                 return -1;
 612         }
 613         if (perf_nb_event_ports(opt) > EVT_MAX_PORTS) {
 614                 evt_err("number of ports exceeds %d", EVT_MAX_PORTS);
 615                 return -1;
 616         }
 617
 618         /* Fixups */
 619         if ((opt->nb_stages == 1 &&
 620                         opt->prod_type != EVT_PROD_TYPE_EVENT_TIMER_ADPTR) &&
 621                         opt->fwd_latency) {
 622                 evt_info("fwd_latency is valid when nb_stages > 1, disabling");
 623                 opt->fwd_latency = 0;
 624         }
 625
 626         if (opt->fwd_latency && !opt->q_priority) {
 627                 evt_info("enabled queue priority for latency measurement");
 628                 opt->q_priority = 1;
 629         }
 630         if (opt->nb_pkts == 0)
 631                 opt->nb_pkts = INT64_MAX/evt_nr_active_lcores(opt->plcores);
 632
 633         return 0;
 634 }
 635
 636 void
 637 perf_opt_dump(struct evt_options *opt, uint8_t nb_queues)
 638 {
 639         evt_dump("nb_prod_lcores", "%d", evt_nr_active_lcores(opt->plcores));
 640         evt_dump_producer_lcores(opt);
 641         evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores));
 642         evt_dump_worker_lcores(opt);
 643         evt_dump_nb_stages(opt);
 644         evt_dump("nb_evdev_ports", "%d", perf_nb_event_ports(opt));
 645         evt_dump("nb_evdev_queues", "%d", nb_queues);
 646         evt_dump_queue_priority(opt);
 647         evt_dump_sched_type_list(opt);
 648         evt_dump_producer_type(opt);
 649 }
 650
 651 void
 652 perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt)
 653 {
 654         int i;
 655         struct test_perf *t = evt_test_priv(test);
 656
 657         if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 658                 for (i = 0; i < opt->nb_timer_adptrs; i++)
 659                         rte_event_timer_adapter_stop(t->timer_adptr[i]);
 660         }
 661         rte_event_dev_stop(opt->dev_id);
 662         rte_event_dev_close(opt->dev_id);
 663 }
 664
 665 static inline void
 666 perf_elt_init(struct rte_mempool *mp, void *arg __rte_unused,
 667             void *obj, unsigned i __rte_unused)
 668 {
 669         memset(obj, 0, mp->elt_size);
 670 }
 671
 672 #define NB_RX_DESC                      128
 673 #define NB_TX_DESC                      512
 674 int
 675 perf_ethdev_setup(struct evt_test *test, struct evt_options *opt)
 676 {
 677         uint16_t i;
 678         struct test_perf *t = evt_test_priv(test);
 679         struct rte_eth_conf port_conf = {
 680                 .rxmode = {
 681                         .mq_mode = ETH_MQ_RX_RSS,
 682                         .max_rx_pkt_len = ETHER_MAX_LEN,
 683                         .split_hdr_size = 0,
 684                 },
 685                 .rx_adv_conf = {
 686                         .rss_conf = {
 687                                 .rss_key = NULL,
 688                                 .rss_hf = ETH_RSS_IP,
 689                         },
 690                 },
 691         };
 692
 693         if (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 694                         opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR)
 695                 return 0;
 696
 697         if (!rte_eth_dev_count_avail()) {
 698                 evt_err("No ethernet ports found.");
 699                 return -ENODEV;
 700         }
 701
 702         RTE_ETH_FOREACH_DEV(i) {
 703                 struct rte_eth_dev_info dev_info;
 704                 struct rte_eth_conf local_port_conf = port_conf;
 705
 706                 rte_eth_dev_info_get(i, &dev_info);
 707
 708                 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
 709                         dev_info.flow_type_rss_offloads;
 710                 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
 711                                 port_conf.rx_adv_conf.rss_conf.rss_hf) {
 712                         evt_info("Port %u modified RSS hash function based on hardware support,"
 713                                 "requested:%#"PRIx64" configured:%#"PRIx64"\n",
 714                                 i,
 715                                 port_conf.rx_adv_conf.rss_conf.rss_hf,
 716                                 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
 717                 }
 718
 719                 if (rte_eth_dev_configure(i, 1, 1, &local_port_conf) < 0) {
 720                         evt_err("Failed to configure eth port [%d]", i);
 721                         return -EINVAL;
 722                 }
 723
 724                 if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC,
 725                                 rte_socket_id(), NULL, t->pool) < 0) {
 726                         evt_err("Failed to setup eth port [%d] rx_queue: %d.",
 727                                         i, 0);
 728                         return -EINVAL;
 729                 }
 730
 731                 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC,
 732                                         rte_socket_id(), NULL) < 0) {
 733                         evt_err("Failed to setup eth port [%d] tx_queue: %d.",
 734                                         i, 0);
 735                         return -EINVAL;
 736                 }
 737
 738                 rte_eth_promiscuous_enable(i);
 739         }
 740
 741         return 0;
 742 }
 743
 744 void perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt)
 745 {
 746         uint16_t i;
 747         RTE_SET_USED(test);
 748
 749         if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) {
 750                 RTE_ETH_FOREACH_DEV(i) {
 751                         rte_event_eth_rx_adapter_stop(i);
 752                         rte_eth_dev_stop(i);
 753                 }
 754         }
 755 }
 756
 757 int
 758 perf_mempool_setup(struct evt_test *test, struct evt_options *opt)
 759 {
 760         struct test_perf *t = evt_test_priv(test);
 761
 762         if (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 763                         opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 764                 t->pool = rte_mempool_create(test->name, /* mempool name */
 765                                 opt->pool_sz, /* number of elements*/
 766                                 sizeof(struct perf_elt), /* element size*/
 767                                 512, /* cache size*/
 768                                 0, NULL, NULL,
 769                                 perf_elt_init, /* obj constructor */
 770                                 NULL, opt->socket_id, 0); /* flags */
 771         } else {
 772                 t->pool = rte_pktmbuf_pool_create(test->name, /* mempool name */
 773                                 opt->pool_sz, /* number of elements*/
 774                                 512, /* cache size*/
 775                                 0,
 776                                 RTE_MBUF_DEFAULT_BUF_SIZE,
 777                                 opt->socket_id); /* flags */
 778
 779         }
 780
 781         if (t->pool == NULL) {
 782                 evt_err("failed to create mempool");
 783                 return -ENOMEM;
 784         }
 785
 786         return 0;
 787 }
 788
 789 void
 790 perf_mempool_destroy(struct evt_test *test, struct evt_options *opt)
 791 {
 792         RTE_SET_USED(opt);
 793         struct test_perf *t = evt_test_priv(test);
 794
 795         rte_mempool_free(t->pool);
 796 }
 797
 798 int
 799 perf_test_setup(struct evt_test *test, struct evt_options *opt)
 800 {
 801         void *test_perf;
 802
 803         test_perf = rte_zmalloc_socket(test->name, sizeof(struct test_perf),
 804                                 RTE_CACHE_LINE_SIZE, opt->socket_id);
 805         if (test_perf  == NULL) {
 806                 evt_err("failed to allocate test_perf memory");
 807                 goto nomem;
 808         }
 809         test->test_priv = test_perf;
 810
 811         struct test_perf *t = evt_test_priv(test);
 812
 813         if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 814                 t->outstand_pkts = opt->nb_timers *
 815                         evt_nr_active_lcores(opt->plcores);
 816                 t->nb_pkts = opt->nb_timers;
 817         } else {
 818                 t->outstand_pkts = opt->nb_pkts *
 819                         evt_nr_active_lcores(opt->plcores);
 820                 t->nb_pkts = opt->nb_pkts;
 821         }
 822
 823         t->nb_workers = evt_nr_active_lcores(opt->wlcores);
 824         t->done = false;
 825         t->nb_flows = opt->nb_flows;
 826         t->result = EVT_TEST_FAILED;
 827         t->opt = opt;
 828         memcpy(t->sched_type_list, opt->sched_type_list,
 829                         sizeof(opt->sched_type_list));
 830         return 0;
 831 nomem:
 832         return -ENOMEM;
 833 }
 834
 835 void
 836 perf_test_destroy(struct evt_test *test, struct evt_options *opt)
 837 {
 838         RTE_SET_USED(opt);
 839
 840         rte_free(test->test_priv);
 841 }