drivers/event/sw/sw_evdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #include <inttypes.h>
   6 #include <string.h>
   7
   8 #include <rte_bus_vdev.h>
   9 #include <rte_kvargs.h>
  10 #include <rte_ring.h>
  11 #include <rte_errno.h>
  12 #include <rte_event_ring.h>
  13 #include <rte_service_component.h>
  14
  15 #include "sw_evdev.h"
  16 #include "iq_chunk.h"
  17
  18 #define EVENTDEV_NAME_SW_PMD event_sw
  19 #define NUMA_NODE_ARG "numa_node"
  20 #define SCHED_QUANTA_ARG "sched_quanta"
  21 #define CREDIT_QUANTA_ARG "credit_quanta"
  22
  23 static void
  24 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info);
  25
  26 static int
  27 sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[],
  28                 const uint8_t priorities[], uint16_t num)
  29 {
  30         struct sw_port *p = port;
  31         struct sw_evdev *sw = sw_pmd_priv(dev);
  32         int i;
  33
  34         RTE_SET_USED(priorities);
  35         for (i = 0; i < num; i++) {
  36                 struct sw_qid *q = &sw->qids[queues[i]];
  37                 unsigned int j;
  38
  39                 /* check for qid map overflow */
  40                 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) {
  41                         rte_errno = -EDQUOT;
  42                         break;
  43                 }
  44
  45                 if (p->is_directed && p->num_qids_mapped > 0) {
  46                         rte_errno = -EDQUOT;
  47                         break;
  48                 }
  49
  50                 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
  51                         if (q->cq_map[j] == p->id)
  52                                 break;
  53                 }
  54
  55                 /* check if port is already linked */
  56                 if (j < q->cq_num_mapped_cqs)
  57                         continue;
  58
  59                 if (q->type == SW_SCHED_TYPE_DIRECT) {
  60                         /* check directed qids only map to one port */
  61                         if (p->num_qids_mapped > 0) {
  62                                 rte_errno = -EDQUOT;
  63                                 break;
  64                         }
  65                         /* check port only takes a directed flow */
  66                         if (num > 1) {
  67                                 rte_errno = -EDQUOT;
  68                                 break;
  69                         }
  70
  71                         p->is_directed = 1;
  72                         p->num_qids_mapped = 1;
  73                 } else if (q->type == RTE_SCHED_TYPE_ORDERED) {
  74                         p->num_ordered_qids++;
  75                         p->num_qids_mapped++;
  76                 } else if (q->type == RTE_SCHED_TYPE_ATOMIC ||
  77                                 q->type == RTE_SCHED_TYPE_PARALLEL) {
  78                         p->num_qids_mapped++;
  79                 }
  80
  81                 q->cq_map[q->cq_num_mapped_cqs] = p->id;
  82                 rte_smp_wmb();
  83                 q->cq_num_mapped_cqs++;
  84         }
  85         return i;
  86 }
  87
  88 static int
  89 sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
  90                 uint16_t nb_unlinks)
  91 {
  92         struct sw_port *p = port;
  93         struct sw_evdev *sw = sw_pmd_priv(dev);
  94         unsigned int i, j;
  95
  96         int unlinked = 0;
  97         for (i = 0; i < nb_unlinks; i++) {
  98                 struct sw_qid *q = &sw->qids[queues[i]];
  99                 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
 100                         if (q->cq_map[j] == p->id) {
 101                                 q->cq_map[j] =
 102                                         q->cq_map[q->cq_num_mapped_cqs - 1];
 103                                 rte_smp_wmb();
 104                                 q->cq_num_mapped_cqs--;
 105                                 unlinked++;
 106
 107                                 p->num_qids_mapped--;
 108
 109                                 if (q->type == RTE_SCHED_TYPE_ORDERED)
 110                                         p->num_ordered_qids--;
 111
 112                                 continue;
 113                         }
 114                 }
 115         }
 116         return unlinked;
 117 }
 118
 119 static int
 120 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 121                 const struct rte_event_port_conf *conf)
 122 {
 123         struct sw_evdev *sw = sw_pmd_priv(dev);
 124         struct sw_port *p = &sw->ports[port_id];
 125         char buf[RTE_RING_NAMESIZE];
 126         unsigned int i;
 127
 128         struct rte_event_dev_info info;
 129         sw_info_get(dev, &info);
 130
 131         /* detect re-configuring and return credits to instance if needed */
 132         if (p->initialized) {
 133                 /* taking credits from pool is done one quanta at a time, and
 134                  * credits may be spend (counted in p->inflights) or still
 135                  * available in the port (p->inflight_credits). We must return
 136                  * the sum to no leak credits
 137                  */
 138                 int possible_inflights = p->inflight_credits + p->inflights;
 139                 rte_atomic32_sub(&sw->inflights, possible_inflights);
 140         }
 141
 142         *p = (struct sw_port){0}; /* zero entire structure */
 143         p->id = port_id;
 144         p->sw = sw;
 145
 146         /* check to see if rings exists - port_setup() can be called multiple
 147          * times legally (assuming device is stopped). If ring exists, free it
 148          * to so it gets re-created with the correct size
 149          */
 150         snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id,
 151                         port_id, "rx_worker_ring");
 152         struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf);
 153         if (existing_ring)
 154                 rte_event_ring_free(existing_ring);
 155
 156         p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH,
 157                         dev->data->socket_id,
 158                         RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
 159         if (p->rx_worker_ring == NULL) {
 160                 SW_LOG_ERR("Error creating RX worker ring for port %d\n",
 161                                 port_id);
 162                 return -1;
 163         }
 164
 165         p->inflight_max = conf->new_event_threshold;
 166         p->implicit_release = !conf->disable_implicit_release;
 167
 168         /* check if ring exists, same as rx_worker above */
 169         snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
 170                         port_id, "cq_worker_ring");
 171         existing_ring = rte_event_ring_lookup(buf);
 172         if (existing_ring)
 173                 rte_event_ring_free(existing_ring);
 174
 175         p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth,
 176                         dev->data->socket_id,
 177                         RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
 178         if (p->cq_worker_ring == NULL) {
 179                 rte_event_ring_free(p->rx_worker_ring);
 180                 SW_LOG_ERR("Error creating CQ worker ring for port %d\n",
 181                                 port_id);
 182                 return -1;
 183         }
 184         sw->cq_ring_space[port_id] = conf->dequeue_depth;
 185
 186         /* set hist list contents to empty */
 187         for (i = 0; i < SW_PORT_HIST_LIST; i++) {
 188                 p->hist_list[i].fid = -1;
 189                 p->hist_list[i].qid = -1;
 190         }
 191         dev->data->ports[port_id] = p;
 192
 193         rte_smp_wmb();
 194         p->initialized = 1;
 195         return 0;
 196 }
 197
 198 static void
 199 sw_port_release(void *port)
 200 {
 201         struct sw_port *p = (void *)port;
 202         if (p == NULL)
 203                 return;
 204
 205         rte_event_ring_free(p->rx_worker_ring);
 206         rte_event_ring_free(p->cq_worker_ring);
 207         memset(p, 0, sizeof(*p));
 208 }
 209
 210 static int32_t
 211 qid_init(struct sw_evdev *sw, unsigned int idx, int type,
 212                 const struct rte_event_queue_conf *queue_conf)
 213 {
 214         unsigned int i;
 215         int dev_id = sw->data->dev_id;
 216         int socket_id = sw->data->socket_id;
 217         char buf[IQ_ROB_NAMESIZE];
 218         struct sw_qid *qid = &sw->qids[idx];
 219
 220         /* Initialize the FID structures to no pinning (-1), and zero packets */
 221         const struct sw_fid_t fid = {.cq = -1, .pcount = 0};
 222         for (i = 0; i < RTE_DIM(qid->fids); i++)
 223                 qid->fids[i] = fid;
 224
 225         qid->id = idx;
 226         qid->type = type;
 227         qid->priority = queue_conf->priority;
 228
 229         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 230                 char ring_name[RTE_RING_NAMESIZE];
 231                 uint32_t window_size;
 232
 233                 /* rte_ring and window_size_mask require require window_size to
 234                  * be a power-of-2.
 235                  */
 236                 window_size = rte_align32pow2(
 237                                 queue_conf->nb_atomic_order_sequences);
 238
 239                 qid->window_size = window_size - 1;
 240
 241                 if (!window_size) {
 242                         SW_LOG_DBG(
 243                                 "invalid reorder_window_size for ordered queue\n"
 244                                 );
 245                         goto cleanup;
 246                 }
 247
 248                 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i);
 249                 qid->reorder_buffer = rte_zmalloc_socket(buf,
 250                                 window_size * sizeof(qid->reorder_buffer[0]),
 251                                 0, socket_id);
 252                 if (!qid->reorder_buffer) {
 253                         SW_LOG_DBG("reorder_buffer malloc failed\n");
 254                         goto cleanup;
 255                 }
 256
 257                 memset(&qid->reorder_buffer[0],
 258                        0,
 259                        window_size * sizeof(qid->reorder_buffer[0]));
 260
 261                 snprintf(ring_name, sizeof(ring_name), "sw%d_q%d_freelist",
 262                                 dev_id, idx);
 263
 264                 /* lookup the ring, and if it already exists, free it */
 265                 struct rte_ring *cleanup = rte_ring_lookup(ring_name);
 266                 if (cleanup)
 267                         rte_ring_free(cleanup);
 268
 269                 qid->reorder_buffer_freelist = rte_ring_create(ring_name,
 270                                 window_size,
 271                                 socket_id,
 272                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
 273                 if (!qid->reorder_buffer_freelist) {
 274                         SW_LOG_DBG("freelist ring create failed");
 275                         goto cleanup;
 276                 }
 277
 278                 /* Populate the freelist with reorder buffer entries. Enqueue
 279                  * 'window_size - 1' entries because the rte_ring holds only
 280                  * that many.
 281                  */
 282                 for (i = 0; i < window_size - 1; i++) {
 283                         if (rte_ring_sp_enqueue(qid->reorder_buffer_freelist,
 284                                                 &qid->reorder_buffer[i]) < 0)
 285                                 goto cleanup;
 286                 }
 287
 288                 qid->reorder_buffer_index = 0;
 289                 qid->cq_next_tx = 0;
 290         }
 291
 292         qid->initialized = 1;
 293
 294         return 0;
 295
 296 cleanup:
 297         if (qid->reorder_buffer) {
 298                 rte_free(qid->reorder_buffer);
 299                 qid->reorder_buffer = NULL;
 300         }
 301
 302         if (qid->reorder_buffer_freelist) {
 303                 rte_ring_free(qid->reorder_buffer_freelist);
 304                 qid->reorder_buffer_freelist = NULL;
 305         }
 306
 307         return -EINVAL;
 308 }
 309
 310 static void
 311 sw_queue_release(struct rte_eventdev *dev, uint8_t id)
 312 {
 313         struct sw_evdev *sw = sw_pmd_priv(dev);
 314         struct sw_qid *qid = &sw->qids[id];
 315
 316         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 317                 rte_free(qid->reorder_buffer);
 318                 rte_ring_free(qid->reorder_buffer_freelist);
 319         }
 320         memset(qid, 0, sizeof(*qid));
 321 }
 322
 323 static int
 324 sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
 325                 const struct rte_event_queue_conf *conf)
 326 {
 327         int type;
 328
 329         type = conf->schedule_type;
 330
 331         if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) {
 332                 type = SW_SCHED_TYPE_DIRECT;
 333         } else if (RTE_EVENT_QUEUE_CFG_ALL_TYPES
 334                         & conf->event_queue_cfg) {
 335                 SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n");
 336                 return -ENOTSUP;
 337         }
 338
 339         struct sw_evdev *sw = sw_pmd_priv(dev);
 340
 341         if (sw->qids[queue_id].initialized)
 342                 sw_queue_release(dev, queue_id);
 343
 344         return qid_init(sw, queue_id, type, conf);
 345 }
 346
 347 static void
 348 sw_init_qid_iqs(struct sw_evdev *sw)
 349 {
 350         int i, j;
 351
 352         /* Initialize the IQ memory of all configured qids */
 353         for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) {
 354                 struct sw_qid *qid = &sw->qids[i];
 355
 356                 if (!qid->initialized)
 357                         continue;
 358
 359                 for (j = 0; j < SW_IQS_MAX; j++)
 360                         iq_init(sw, &qid->iq[j]);
 361         }
 362 }
 363
 364 static int
 365 sw_qids_empty(struct sw_evdev *sw)
 366 {
 367         unsigned int i, j;
 368
 369         for (i = 0; i < sw->qid_count; i++) {
 370                 for (j = 0; j < SW_IQS_MAX; j++) {
 371                         if (iq_count(&sw->qids[i].iq[j]))
 372                                 return 0;
 373                 }
 374         }
 375
 376         return 1;
 377 }
 378
 379 static int
 380 sw_ports_empty(struct sw_evdev *sw)
 381 {
 382         unsigned int i;
 383
 384         for (i = 0; i < sw->port_count; i++) {
 385                 if ((rte_event_ring_count(sw->ports[i].rx_worker_ring)) ||
 386                      rte_event_ring_count(sw->ports[i].cq_worker_ring))
 387                         return 0;
 388         }
 389
 390         return 1;
 391 }
 392
 393 static void
 394 sw_drain_ports(struct rte_eventdev *dev)
 395 {
 396         struct sw_evdev *sw = sw_pmd_priv(dev);
 397         eventdev_stop_flush_t flush;
 398         unsigned int i;
 399         uint8_t dev_id;
 400         void *arg;
 401
 402         flush = dev->dev_ops->dev_stop_flush;
 403         dev_id = dev->data->dev_id;
 404         arg = dev->data->dev_stop_flush_arg;
 405
 406         for (i = 0; i < sw->port_count; i++) {
 407                 struct rte_event ev;
 408
 409                 while (rte_event_dequeue_burst(dev_id, i, &ev, 1, 0)) {
 410                         if (flush)
 411                                 flush(dev_id, ev, arg);
 412
 413                         ev.op = RTE_EVENT_OP_RELEASE;
 414                         rte_event_enqueue_burst(dev_id, i, &ev, 1);
 415                 }
 416         }
 417 }
 418
 419 static void
 420 sw_drain_queue(struct rte_eventdev *dev, struct sw_iq *iq)
 421 {
 422         struct sw_evdev *sw = sw_pmd_priv(dev);
 423         eventdev_stop_flush_t flush;
 424         uint8_t dev_id;
 425         void *arg;
 426
 427         flush = dev->dev_ops->dev_stop_flush;
 428         dev_id = dev->data->dev_id;
 429         arg = dev->data->dev_stop_flush_arg;
 430
 431         while (iq_count(iq) > 0) {
 432                 struct rte_event ev;
 433
 434                 iq_dequeue_burst(sw, iq, &ev, 1);
 435
 436                 if (flush)
 437                         flush(dev_id, ev, arg);
 438         }
 439 }
 440
 441 static void
 442 sw_drain_queues(struct rte_eventdev *dev)
 443 {
 444         struct sw_evdev *sw = sw_pmd_priv(dev);
 445         unsigned int i, j;
 446
 447         for (i = 0; i < sw->qid_count; i++) {
 448                 for (j = 0; j < SW_IQS_MAX; j++)
 449                         sw_drain_queue(dev, &sw->qids[i].iq[j]);
 450         }
 451 }
 452
 453 static void
 454 sw_clean_qid_iqs(struct rte_eventdev *dev)
 455 {
 456         struct sw_evdev *sw = sw_pmd_priv(dev);
 457         int i, j;
 458
 459         /* Release the IQ memory of all configured qids */
 460         for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) {
 461                 struct sw_qid *qid = &sw->qids[i];
 462
 463                 for (j = 0; j < SW_IQS_MAX; j++) {
 464                         if (!qid->iq[j].head)
 465                                 continue;
 466                         iq_free_chunk_list(sw, qid->iq[j].head);
 467                         qid->iq[j].head = NULL;
 468                 }
 469         }
 470 }
 471
 472 static void
 473 sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
 474                                  struct rte_event_queue_conf *conf)
 475 {
 476         RTE_SET_USED(dev);
 477         RTE_SET_USED(queue_id);
 478
 479         static const struct rte_event_queue_conf default_conf = {
 480                 .nb_atomic_flows = 4096,
 481                 .nb_atomic_order_sequences = 1,
 482                 .schedule_type = RTE_SCHED_TYPE_ATOMIC,
 483                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 484         };
 485
 486         *conf = default_conf;
 487 }
 488
 489 static void
 490 sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 491                  struct rte_event_port_conf *port_conf)
 492 {
 493         RTE_SET_USED(dev);
 494         RTE_SET_USED(port_id);
 495
 496         port_conf->new_event_threshold = 1024;
 497         port_conf->dequeue_depth = 16;
 498         port_conf->enqueue_depth = 16;
 499         port_conf->disable_implicit_release = 0;
 500 }
 501
 502 static int
 503 sw_dev_configure(const struct rte_eventdev *dev)
 504 {
 505         struct sw_evdev *sw = sw_pmd_priv(dev);
 506         const struct rte_eventdev_data *data = dev->data;
 507         const struct rte_event_dev_config *conf = &data->dev_conf;
 508         int num_chunks, i;
 509
 510         sw->qid_count = conf->nb_event_queues;
 511         sw->port_count = conf->nb_event_ports;
 512         sw->nb_events_limit = conf->nb_events_limit;
 513         rte_atomic32_set(&sw->inflights, 0);
 514
 515         /* Number of chunks sized for worst-case spread of events across IQs */
 516         num_chunks = ((SW_INFLIGHT_EVENTS_TOTAL/SW_EVS_PER_Q_CHUNK)+1) +
 517                         sw->qid_count*SW_IQS_MAX*2;
 518
 519         /* If this is a reconfiguration, free the previous IQ allocation. All
 520          * IQ chunk references were cleaned out of the QIDs in sw_stop(), and
 521          * will be reinitialized in sw_start().
 522          */
 523         if (sw->chunks)
 524                 rte_free(sw->chunks);
 525
 526         sw->chunks = rte_malloc_socket(NULL,
 527                                        sizeof(struct sw_queue_chunk) *
 528                                        num_chunks,
 529                                        0,
 530                                        sw->data->socket_id);
 531         if (!sw->chunks)
 532                 return -ENOMEM;
 533
 534         sw->chunk_list_head = NULL;
 535         for (i = 0; i < num_chunks; i++)
 536                 iq_free_chunk(sw, &sw->chunks[i]);
 537
 538         if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
 539                 return -ENOTSUP;
 540
 541         return 0;
 542 }
 543
 544 struct rte_eth_dev;
 545
 546 static int
 547 sw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev,
 548                         const struct rte_eth_dev *eth_dev,
 549                         uint32_t *caps)
 550 {
 551         RTE_SET_USED(dev);
 552         RTE_SET_USED(eth_dev);
 553         *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP;
 554         return 0;
 555 }
 556
 557 static int
 558 sw_timer_adapter_caps_get(const struct rte_eventdev *dev,
 559                           uint64_t flags,
 560                           uint32_t *caps,
 561                           const struct rte_event_timer_adapter_ops **ops)
 562 {
 563         RTE_SET_USED(dev);
 564         RTE_SET_USED(flags);
 565         *caps = 0;
 566
 567         /* Use default SW ops */
 568         *ops = NULL;
 569
 570         return 0;
 571 }
 572
 573 static int
 574 sw_crypto_adapter_caps_get(const struct rte_eventdev *dev,
 575                            const struct rte_cryptodev *cdev,
 576                            uint32_t *caps)
 577 {
 578         RTE_SET_USED(dev);
 579         RTE_SET_USED(cdev);
 580         *caps = RTE_EVENT_CRYPTO_ADAPTER_SW_CAP;
 581         return 0;
 582 }
 583
 584 static void
 585 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info)
 586 {
 587         RTE_SET_USED(dev);
 588
 589         static const struct rte_event_dev_info evdev_sw_info = {
 590                         .driver_name = SW_PMD_NAME,
 591                         .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
 592                         .max_event_queue_flows = SW_QID_NUM_FIDS,
 593                         .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX,
 594                         .max_event_priority_levels = SW_IQS_MAX,
 595                         .max_event_ports = SW_PORTS_MAX,
 596                         .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH,
 597                         .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH,
 598                         .max_num_events = SW_INFLIGHT_EVENTS_TOTAL,
 599                         .event_dev_cap = (
 600                                 RTE_EVENT_DEV_CAP_QUEUE_QOS |
 601                                 RTE_EVENT_DEV_CAP_BURST_MODE |
 602                                 RTE_EVENT_DEV_CAP_EVENT_QOS |
 603                                 RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE|
 604                                 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK |
 605                                 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT |
 606                                 RTE_EVENT_DEV_CAP_NONSEQ_MODE),
 607         };
 608
 609         *info = evdev_sw_info;
 610 }
 611
 612 static void
 613 sw_dump(struct rte_eventdev *dev, FILE *f)
 614 {
 615         const struct sw_evdev *sw = sw_pmd_priv(dev);
 616
 617         static const char * const q_type_strings[] = {
 618                         "Ordered", "Atomic", "Parallel", "Directed"
 619         };
 620         uint32_t i;
 621         fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name",
 622                         sw->port_count, sw->qid_count);
 623
 624         fprintf(f, "\trx   %"PRIu64"\n\tdrop %"PRIu64"\n\ttx   %"PRIu64"\n",
 625                 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts);
 626         fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called);
 627         fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called);
 628         fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues);
 629         fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues);
 630         uint32_t inflights = rte_atomic32_read(&sw->inflights);
 631         uint32_t credits = sw->nb_events_limit - inflights;
 632         fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits);
 633
 634 #define COL_RED "\x1b[31m"
 635 #define COL_RESET "\x1b[0m"
 636
 637         for (i = 0; i < sw->port_count; i++) {
 638                 int max, j;
 639                 const struct sw_port *p = &sw->ports[i];
 640                 if (!p->initialized) {
 641                         fprintf(f, "  %sPort %d not initialized.%s\n",
 642                                 COL_RED, i, COL_RESET);
 643                         continue;
 644                 }
 645                 fprintf(f, "  Port %d %s\n", i,
 646                         p->is_directed ? " (SingleCons)" : "");
 647                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64
 648                         "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts,
 649                         sw->ports[i].stats.rx_dropped,
 650                         sw->ports[i].stats.tx_pkts,
 651                         (p->inflights == p->inflight_max) ?
 652                                 COL_RED : COL_RESET,
 653                         sw->ports[i].inflights, COL_RESET);
 654
 655                 fprintf(f, "\tMax New: %u"
 656                         "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n",
 657                         sw->ports[i].inflight_max,
 658                         sw->ports[i].avg_pkt_ticks,
 659                         sw->ports[i].inflight_credits);
 660                 fprintf(f, "\tReceive burst distribution:\n");
 661                 float zp_percent = p->zero_polls * 100.0 / p->total_polls;
 662                 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ",
 663                                 zp_percent);
 664                 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;)
 665                         if (p->poll_buckets[max] != 0)
 666                                 break;
 667                 for (j = 0; j <= max; j++) {
 668                         if (p->poll_buckets[j] != 0) {
 669                                 float poll_pc = p->poll_buckets[j] * 100.0 /
 670                                         p->total_polls;
 671                                 fprintf(f, "%u-%u:%.02f%% ",
 672                                         ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1),
 673                                         ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT),
 674                                         poll_pc);
 675                         }
 676                 }
 677                 fprintf(f, "\n");
 678
 679                 if (p->rx_worker_ring) {
 680                         uint64_t used = rte_event_ring_count(p->rx_worker_ring);
 681                         uint64_t space = rte_event_ring_free_count(
 682                                         p->rx_worker_ring);
 683                         const char *col = (space == 0) ? COL_RED : COL_RESET;
 684                         fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4"
 685                                         PRIu64 COL_RESET"\n", col, used, space);
 686                 } else
 687                         fprintf(f, "\trx ring not initialized.\n");
 688
 689                 if (p->cq_worker_ring) {
 690                         uint64_t used = rte_event_ring_count(p->cq_worker_ring);
 691                         uint64_t space = rte_event_ring_free_count(
 692                                         p->cq_worker_ring);
 693                         const char *col = (space == 0) ? COL_RED : COL_RESET;
 694                         fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4"
 695                                         PRIu64 COL_RESET"\n", col, used, space);
 696                 } else
 697                         fprintf(f, "\tcq ring not initialized.\n");
 698         }
 699
 700         for (i = 0; i < sw->qid_count; i++) {
 701                 const struct sw_qid *qid = &sw->qids[i];
 702                 if (!qid->initialized) {
 703                         fprintf(f, "  %sQueue %d not initialized.%s\n",
 704                                 COL_RED, i, COL_RESET);
 705                         continue;
 706                 }
 707                 int affinities_per_port[SW_PORTS_MAX] = {0};
 708                 uint32_t inflights = 0;
 709
 710                 fprintf(f, "  Queue %d (%s)\n", i, q_type_strings[qid->type]);
 711                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64"\n",
 712                         qid->stats.rx_pkts, qid->stats.rx_dropped,
 713                         qid->stats.tx_pkts);
 714                 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 715                         struct rte_ring *rob_buf_free =
 716                                 qid->reorder_buffer_freelist;
 717                         if (rob_buf_free)
 718                                 fprintf(f, "\tReorder entries in use: %u\n",
 719                                         rte_ring_free_count(rob_buf_free));
 720                         else
 721                                 fprintf(f,
 722                                         "\tReorder buffer not initialized\n");
 723                 }
 724
 725                 uint32_t flow;
 726                 for (flow = 0; flow < RTE_DIM(qid->fids); flow++)
 727                         if (qid->fids[flow].cq != -1) {
 728                                 affinities_per_port[qid->fids[flow].cq]++;
 729                                 inflights += qid->fids[flow].pcount;
 730                         }
 731
 732                 uint32_t port;
 733                 fprintf(f, "\tPer Port Stats:\n");
 734                 for (port = 0; port < sw->port_count; port++) {
 735                         fprintf(f, "\t  Port %d: Pkts: %"PRIu64, port,
 736                                         qid->to_port[port]);
 737                         fprintf(f, "\tFlows: %d\n", affinities_per_port[port]);
 738                 }
 739
 740                 uint32_t iq;
 741                 uint32_t iq_printed = 0;
 742                 for (iq = 0; iq < SW_IQS_MAX; iq++) {
 743                         if (!qid->iq[iq].head) {
 744                                 fprintf(f, "\tiq %d is not initialized.\n", iq);
 745                                 iq_printed = 1;
 746                                 continue;
 747                         }
 748                         uint32_t used = iq_count(&qid->iq[iq]);
 749                         const char *col = COL_RESET;
 750                         if (used > 0) {
 751                                 fprintf(f, "\t%siq %d: Used %d"
 752                                         COL_RESET"\n", col, iq, used);
 753                                 iq_printed = 1;
 754                         }
 755                 }
 756                 if (iq_printed == 0)
 757                         fprintf(f, "\t-- iqs empty --\n");
 758         }
 759 }
 760
 761 static int
 762 sw_start(struct rte_eventdev *dev)
 763 {
 764         unsigned int i, j;
 765         struct sw_evdev *sw = sw_pmd_priv(dev);
 766
 767         rte_service_component_runstate_set(sw->service_id, 1);
 768
 769         /* check a service core is mapped to this service */
 770         if (!rte_service_runstate_get(sw->service_id)) {
 771                 SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
 772                                 sw->service_name);
 773                 return -ENOENT;
 774         }
 775
 776         /* check all ports are set up */
 777         for (i = 0; i < sw->port_count; i++)
 778                 if (sw->ports[i].rx_worker_ring == NULL) {
 779                         SW_LOG_ERR("Port %d not configured\n", i);
 780                         return -ESTALE;
 781                 }
 782
 783         /* check all queues are configured and mapped to ports*/
 784         for (i = 0; i < sw->qid_count; i++)
 785                 if (!sw->qids[i].initialized ||
 786                     sw->qids[i].cq_num_mapped_cqs == 0) {
 787                         SW_LOG_ERR("Queue %d not configured\n", i);
 788                         return -ENOLINK;
 789                 }
 790
 791         /* build up our prioritized array of qids */
 792         /* We don't use qsort here, as if all/multiple entries have the same
 793          * priority, the result is non-deterministic. From "man 3 qsort":
 794          * "If two members compare as equal, their order in the sorted
 795          * array is undefined."
 796          */
 797         uint32_t qidx = 0;
 798         for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) {
 799                 for (i = 0; i < sw->qid_count; i++) {
 800                         if (sw->qids[i].priority == j) {
 801                                 sw->qids_prioritized[qidx] = &sw->qids[i];
 802                                 qidx++;
 803                         }
 804                 }
 805         }
 806
 807         sw_init_qid_iqs(sw);
 808
 809         if (sw_xstats_init(sw) < 0)
 810                 return -EINVAL;
 811
 812         rte_smp_wmb();
 813         sw->started = 1;
 814
 815         return 0;
 816 }
 817
 818 static void
 819 sw_stop(struct rte_eventdev *dev)
 820 {
 821         struct sw_evdev *sw = sw_pmd_priv(dev);
 822         int32_t runstate;
 823
 824         /* Stop the scheduler if it's running */
 825         runstate = rte_service_runstate_get(sw->service_id);
 826         if (runstate == 1)
 827                 rte_service_runstate_set(sw->service_id, 0);
 828
 829         while (rte_service_may_be_active(sw->service_id))
 830                 rte_pause();
 831
 832         /* Flush all events out of the device */
 833         while (!(sw_qids_empty(sw) && sw_ports_empty(sw))) {
 834                 sw_event_schedule(dev);
 835                 sw_drain_ports(dev);
 836                 sw_drain_queues(dev);
 837         }
 838
 839         sw_clean_qid_iqs(dev);
 840         sw_xstats_uninit(sw);
 841         sw->started = 0;
 842         rte_smp_wmb();
 843
 844         if (runstate == 1)
 845                 rte_service_runstate_set(sw->service_id, 1);
 846 }
 847
 848 static int
 849 sw_close(struct rte_eventdev *dev)
 850 {
 851         struct sw_evdev *sw = sw_pmd_priv(dev);
 852         uint32_t i;
 853
 854         for (i = 0; i < sw->qid_count; i++)
 855                 sw_queue_release(dev, i);
 856         sw->qid_count = 0;
 857
 858         for (i = 0; i < sw->port_count; i++)
 859                 sw_port_release(&sw->ports[i]);
 860         sw->port_count = 0;
 861
 862         memset(&sw->stats, 0, sizeof(sw->stats));
 863         sw->sched_called = 0;
 864         sw->sched_no_iq_enqueues = 0;
 865         sw->sched_no_cq_enqueues = 0;
 866         sw->sched_cq_qid_called = 0;
 867
 868         return 0;
 869 }
 870
 871 static int
 872 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque)
 873 {
 874         int *socket_id = opaque;
 875         *socket_id = atoi(value);
 876         if (*socket_id >= RTE_MAX_NUMA_NODES)
 877                 return -1;
 878         return 0;
 879 }
 880
 881 static int
 882 set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque)
 883 {
 884         int *quanta = opaque;
 885         *quanta = atoi(value);
 886         if (*quanta < 0 || *quanta >= 4096)
 887                 return -1;
 888         return 0;
 889 }
 890
 891 static int
 892 set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
 893 {
 894         int *credit = opaque;
 895         *credit = atoi(value);
 896         if (*credit < 0 || *credit >= 128)
 897                 return -1;
 898         return 0;
 899 }
 900
 901
 902 static int32_t sw_sched_service_func(void *args)
 903 {
 904         struct rte_eventdev *dev = args;
 905         sw_event_schedule(dev);
 906         return 0;
 907 }
 908
 909 static int
 910 sw_probe(struct rte_vdev_device *vdev)
 911 {
 912         static struct rte_eventdev_ops evdev_sw_ops = {
 913                         .dev_configure = sw_dev_configure,
 914                         .dev_infos_get = sw_info_get,
 915                         .dev_close = sw_close,
 916                         .dev_start = sw_start,
 917                         .dev_stop = sw_stop,
 918                         .dump = sw_dump,
 919
 920                         .queue_def_conf = sw_queue_def_conf,
 921                         .queue_setup = sw_queue_setup,
 922                         .queue_release = sw_queue_release,
 923                         .port_def_conf = sw_port_def_conf,
 924                         .port_setup = sw_port_setup,
 925                         .port_release = sw_port_release,
 926                         .port_link = sw_port_link,
 927                         .port_unlink = sw_port_unlink,
 928
 929                         .eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get,
 930
 931                         .timer_adapter_caps_get = sw_timer_adapter_caps_get,
 932
 933                         .crypto_adapter_caps_get = sw_crypto_adapter_caps_get,
 934
 935                         .xstats_get = sw_xstats_get,
 936                         .xstats_get_names = sw_xstats_get_names,
 937                         .xstats_get_by_name = sw_xstats_get_by_name,
 938                         .xstats_reset = sw_xstats_reset,
 939
 940                         .dev_selftest = test_sw_eventdev,
 941         };
 942
 943         static const char *const args[] = {
 944                 NUMA_NODE_ARG,
 945                 SCHED_QUANTA_ARG,
 946                 CREDIT_QUANTA_ARG,
 947                 NULL
 948         };
 949         const char *name;
 950         const char *params;
 951         struct rte_eventdev *dev;
 952         struct sw_evdev *sw;
 953         int socket_id = rte_socket_id();
 954         int sched_quanta  = SW_DEFAULT_SCHED_QUANTA;
 955         int credit_quanta = SW_DEFAULT_CREDIT_QUANTA;
 956
 957         name = rte_vdev_device_name(vdev);
 958         params = rte_vdev_device_args(vdev);
 959         if (params != NULL && params[0] != '\0') {
 960                 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
 961
 962                 if (!kvlist) {
 963                         SW_LOG_INFO(
 964                                 "Ignoring unsupported parameters when creating device '%s'\n",
 965                                 name);
 966                 } else {
 967                         int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
 968                                         assign_numa_node, &socket_id);
 969                         if (ret != 0) {
 970                                 SW_LOG_ERR(
 971                                         "%s: Error parsing numa node parameter",
 972                                         name);
 973                                 rte_kvargs_free(kvlist);
 974                                 return ret;
 975                         }
 976
 977                         ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG,
 978                                         set_sched_quanta, &sched_quanta);
 979                         if (ret != 0) {
 980                                 SW_LOG_ERR(
 981                                         "%s: Error parsing sched quanta parameter",
 982                                         name);
 983                                 rte_kvargs_free(kvlist);
 984                                 return ret;
 985                         }
 986
 987                         ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG,
 988                                         set_credit_quanta, &credit_quanta);
 989                         if (ret != 0) {
 990                                 SW_LOG_ERR(
 991                                         "%s: Error parsing credit quanta parameter",
 992                                         name);
 993                                 rte_kvargs_free(kvlist);
 994                                 return ret;
 995                         }
 996
 997                         rte_kvargs_free(kvlist);
 998                 }
 999         }
1000
1001         SW_LOG_INFO(
1002                         "Creating eventdev sw device %s, numa_node=%d, sched_quanta=%d, credit_quanta=%d\n",
1003                         name, socket_id, sched_quanta, credit_quanta);
1004
1005         dev = rte_event_pmd_vdev_init(name,
1006                         sizeof(struct sw_evdev), socket_id);
1007         if (dev == NULL) {
1008                 SW_LOG_ERR("eventdev vdev init() failed");
1009                 return -EFAULT;
1010         }
1011         dev->dev_ops = &evdev_sw_ops;
1012         dev->enqueue = sw_event_enqueue;
1013         dev->enqueue_burst = sw_event_enqueue_burst;
1014         dev->enqueue_new_burst = sw_event_enqueue_burst;
1015         dev->enqueue_forward_burst = sw_event_enqueue_burst;
1016         dev->dequeue = sw_event_dequeue;
1017         dev->dequeue_burst = sw_event_dequeue_burst;
1018
1019         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1020                 return 0;
1021
1022         sw = dev->data->dev_private;
1023         sw->data = dev->data;
1024
1025         /* copy values passed from vdev command line to instance */
1026         sw->credit_update_quanta = credit_quanta;
1027         sw->sched_quanta = sched_quanta;
1028
1029         /* register service with EAL */
1030         struct rte_service_spec service;
1031         memset(&service, 0, sizeof(struct rte_service_spec));
1032         snprintf(service.name, sizeof(service.name), "%s_service", name);
1033         snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
1034                         name);
1035         service.socket_id = socket_id;
1036         service.callback = sw_sched_service_func;
1037         service.callback_userdata = (void *)dev;
1038
1039         int32_t ret = rte_service_component_register(&service, &sw->service_id);
1040         if (ret) {
1041                 SW_LOG_ERR("service register() failed");
1042                 return -ENOEXEC;
1043         }
1044
1045         dev->data->service_inited = 1;
1046         dev->data->service_id = sw->service_id;
1047
1048         return 0;
1049 }
1050
1051 static int
1052 sw_remove(struct rte_vdev_device *vdev)
1053 {
1054         const char *name;
1055
1056         name = rte_vdev_device_name(vdev);
1057         if (name == NULL)
1058                 return -EINVAL;
1059
1060         SW_LOG_INFO("Closing eventdev sw device %s\n", name);
1061
1062         return rte_event_pmd_vdev_uninit(name);
1063 }
1064
1065 static struct rte_vdev_driver evdev_sw_pmd_drv = {
1066         .probe = sw_probe,
1067         .remove = sw_remove
1068 };
1069
1070 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv);
1071 RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> "
1072                 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>");
1073
1074 /* declared extern in header, for access from other .c files */
1075 int eventdev_sw_log_level;
1076
1077 RTE_INIT(evdev_sw_init_log)
1078 {
1079         eventdev_sw_log_level = rte_log_register("pmd.event.sw");
1080         if (eventdev_sw_log_level >= 0)
1081                 rte_log_set_level(eventdev_sw_log_level, RTE_LOG_NOTICE);
1082 }