drivers/event/sw/sw_evdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2016-2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <inttypes.h>
  34 #include <string.h>
  35
  36 #include <rte_vdev.h>
  37 #include <rte_memzone.h>
  38 #include <rte_kvargs.h>
  39 #include <rte_ring.h>
  40 #include <rte_errno.h>
  41 #include <rte_event_ring.h>
  42 #include <rte_service_component.h>
  43
  44 #include "sw_evdev.h"
  45 #include "iq_ring.h"
  46
  47 #define EVENTDEV_NAME_SW_PMD event_sw
  48 #define NUMA_NODE_ARG "numa_node"
  49 #define SCHED_QUANTA_ARG "sched_quanta"
  50 #define CREDIT_QUANTA_ARG "credit_quanta"
  51
  52 static void
  53 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info);
  54
  55 static int
  56 sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[],
  57                 const uint8_t priorities[], uint16_t num)
  58 {
  59         struct sw_port *p = port;
  60         struct sw_evdev *sw = sw_pmd_priv(dev);
  61         int i;
  62
  63         RTE_SET_USED(priorities);
  64         for (i = 0; i < num; i++) {
  65                 struct sw_qid *q = &sw->qids[queues[i]];
  66
  67                 /* check for qid map overflow */
  68                 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) {
  69                         rte_errno = -EDQUOT;
  70                         break;
  71                 }
  72
  73                 if (p->is_directed && p->num_qids_mapped > 0) {
  74                         rte_errno = -EDQUOT;
  75                         break;
  76                 }
  77
  78                 if (q->type == SW_SCHED_TYPE_DIRECT) {
  79                         /* check directed qids only map to one port */
  80                         if (p->num_qids_mapped > 0) {
  81                                 rte_errno = -EDQUOT;
  82                                 break;
  83                         }
  84                         /* check port only takes a directed flow */
  85                         if (num > 1) {
  86                                 rte_errno = -EDQUOT;
  87                                 break;
  88                         }
  89
  90                         p->is_directed = 1;
  91                         p->num_qids_mapped = 1;
  92                 } else if (q->type == RTE_SCHED_TYPE_ORDERED) {
  93                         p->num_ordered_qids++;
  94                         p->num_qids_mapped++;
  95                 } else if (q->type == RTE_SCHED_TYPE_ATOMIC ||
  96                                 q->type == RTE_SCHED_TYPE_PARALLEL) {
  97                         p->num_qids_mapped++;
  98                 }
  99
 100                 q->cq_map[q->cq_num_mapped_cqs] = p->id;
 101                 rte_smp_wmb();
 102                 q->cq_num_mapped_cqs++;
 103         }
 104         return i;
 105 }
 106
 107 static int
 108 sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
 109                 uint16_t nb_unlinks)
 110 {
 111         struct sw_port *p = port;
 112         struct sw_evdev *sw = sw_pmd_priv(dev);
 113         unsigned int i, j;
 114
 115         int unlinked = 0;
 116         for (i = 0; i < nb_unlinks; i++) {
 117                 struct sw_qid *q = &sw->qids[queues[i]];
 118                 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
 119                         if (q->cq_map[j] == p->id) {
 120                                 q->cq_map[j] =
 121                                         q->cq_map[q->cq_num_mapped_cqs - 1];
 122                                 rte_smp_wmb();
 123                                 q->cq_num_mapped_cqs--;
 124                                 unlinked++;
 125
 126                                 p->num_qids_mapped--;
 127
 128                                 if (q->type == RTE_SCHED_TYPE_ORDERED)
 129                                         p->num_ordered_qids--;
 130
 131                                 continue;
 132                         }
 133                 }
 134         }
 135         return unlinked;
 136 }
 137
 138 static int
 139 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 140                 const struct rte_event_port_conf *conf)
 141 {
 142         struct sw_evdev *sw = sw_pmd_priv(dev);
 143         struct sw_port *p = &sw->ports[port_id];
 144         char buf[RTE_RING_NAMESIZE];
 145         unsigned int i;
 146
 147         struct rte_event_dev_info info;
 148         sw_info_get(dev, &info);
 149
 150         /* detect re-configuring and return credits to instance if needed */
 151         if (p->initialized) {
 152                 /* taking credits from pool is done one quanta at a time, and
 153                  * credits may be spend (counted in p->inflights) or still
 154                  * available in the port (p->inflight_credits). We must return
 155                  * the sum to no leak credits
 156                  */
 157                 int possible_inflights = p->inflight_credits + p->inflights;
 158                 rte_atomic32_sub(&sw->inflights, possible_inflights);
 159         }
 160
 161         *p = (struct sw_port){0}; /* zero entire structure */
 162         p->id = port_id;
 163         p->sw = sw;
 164
 165         /* check to see if rings exists - port_setup() can be called multiple
 166          * times legally (assuming device is stopped). If ring exists, free it
 167          * to so it gets re-created with the correct size
 168          */
 169         snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id,
 170                         port_id, "rx_worker_ring");
 171         struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf);
 172         if (existing_ring)
 173                 rte_event_ring_free(existing_ring);
 174
 175         p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH,
 176                         dev->data->socket_id,
 177                         RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
 178         if (p->rx_worker_ring == NULL) {
 179                 SW_LOG_ERR("Error creating RX worker ring for port %d\n",
 180                                 port_id);
 181                 return -1;
 182         }
 183
 184         p->inflight_max = conf->new_event_threshold;
 185
 186         /* check if ring exists, same as rx_worker above */
 187         snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
 188                         port_id, "cq_worker_ring");
 189         existing_ring = rte_event_ring_lookup(buf);
 190         if (existing_ring)
 191                 rte_event_ring_free(existing_ring);
 192
 193         p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth,
 194                         dev->data->socket_id,
 195                         RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
 196         if (p->cq_worker_ring == NULL) {
 197                 rte_event_ring_free(p->rx_worker_ring);
 198                 SW_LOG_ERR("Error creating CQ worker ring for port %d\n",
 199                                 port_id);
 200                 return -1;
 201         }
 202         sw->cq_ring_space[port_id] = conf->dequeue_depth;
 203
 204         /* set hist list contents to empty */
 205         for (i = 0; i < SW_PORT_HIST_LIST; i++) {
 206                 p->hist_list[i].fid = -1;
 207                 p->hist_list[i].qid = -1;
 208         }
 209         dev->data->ports[port_id] = p;
 210
 211         rte_smp_wmb();
 212         p->initialized = 1;
 213         return 0;
 214 }
 215
 216 static void
 217 sw_port_release(void *port)
 218 {
 219         struct sw_port *p = (void *)port;
 220         if (p == NULL)
 221                 return;
 222
 223         rte_event_ring_free(p->rx_worker_ring);
 224         rte_event_ring_free(p->cq_worker_ring);
 225         memset(p, 0, sizeof(*p));
 226 }
 227
 228 static int32_t
 229 qid_init(struct sw_evdev *sw, unsigned int idx, int type,
 230                 const struct rte_event_queue_conf *queue_conf)
 231 {
 232         unsigned int i;
 233         int dev_id = sw->data->dev_id;
 234         int socket_id = sw->data->socket_id;
 235         char buf[IQ_RING_NAMESIZE];
 236         struct sw_qid *qid = &sw->qids[idx];
 237
 238         for (i = 0; i < SW_IQS_MAX; i++) {
 239                 snprintf(buf, sizeof(buf), "q_%u_iq_%d", idx, i);
 240                 qid->iq[i] = iq_ring_create(buf, socket_id);
 241                 if (!qid->iq[i]) {
 242                         SW_LOG_DBG("ring create failed");
 243                         goto cleanup;
 244                 }
 245         }
 246
 247         /* Initialize the FID structures to no pinning (-1), and zero packets */
 248         const struct sw_fid_t fid = {.cq = -1, .pcount = 0};
 249         for (i = 0; i < RTE_DIM(qid->fids); i++)
 250                 qid->fids[i] = fid;
 251
 252         qid->id = idx;
 253         qid->type = type;
 254         qid->priority = queue_conf->priority;
 255
 256         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 257                 char ring_name[RTE_RING_NAMESIZE];
 258                 uint32_t window_size;
 259
 260                 /* rte_ring and window_size_mask require require window_size to
 261                  * be a power-of-2.
 262                  */
 263                 window_size = rte_align32pow2(
 264                                 queue_conf->nb_atomic_order_sequences);
 265
 266                 qid->window_size = window_size - 1;
 267
 268                 if (!window_size) {
 269                         SW_LOG_DBG(
 270                                 "invalid reorder_window_size for ordered queue\n"
 271                                 );
 272                         goto cleanup;
 273                 }
 274
 275                 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i);
 276                 qid->reorder_buffer = rte_zmalloc_socket(buf,
 277                                 window_size * sizeof(qid->reorder_buffer[0]),
 278                                 0, socket_id);
 279                 if (!qid->reorder_buffer) {
 280                         SW_LOG_DBG("reorder_buffer malloc failed\n");
 281                         goto cleanup;
 282                 }
 283
 284                 memset(&qid->reorder_buffer[0],
 285                        0,
 286                        window_size * sizeof(qid->reorder_buffer[0]));
 287
 288                 snprintf(ring_name, sizeof(ring_name), "sw%d_q%d_freelist",
 289                                 dev_id, idx);
 290
 291                 /* lookup the ring, and if it already exists, free it */
 292                 struct rte_ring *cleanup = rte_ring_lookup(ring_name);
 293                 if (cleanup)
 294                         rte_ring_free(cleanup);
 295
 296                 qid->reorder_buffer_freelist = rte_ring_create(ring_name,
 297                                 window_size,
 298                                 socket_id,
 299                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
 300                 if (!qid->reorder_buffer_freelist) {
 301                         SW_LOG_DBG("freelist ring create failed");
 302                         goto cleanup;
 303                 }
 304
 305                 /* Populate the freelist with reorder buffer entries. Enqueue
 306                  * 'window_size - 1' entries because the rte_ring holds only
 307                  * that many.
 308                  */
 309                 for (i = 0; i < window_size - 1; i++) {
 310                         if (rte_ring_sp_enqueue(qid->reorder_buffer_freelist,
 311                                                 &qid->reorder_buffer[i]) < 0)
 312                                 goto cleanup;
 313                 }
 314
 315                 qid->reorder_buffer_index = 0;
 316                 qid->cq_next_tx = 0;
 317         }
 318
 319         qid->initialized = 1;
 320
 321         return 0;
 322
 323 cleanup:
 324         for (i = 0; i < SW_IQS_MAX; i++) {
 325                 if (qid->iq[i])
 326                         iq_ring_destroy(qid->iq[i]);
 327         }
 328
 329         if (qid->reorder_buffer) {
 330                 rte_free(qid->reorder_buffer);
 331                 qid->reorder_buffer = NULL;
 332         }
 333
 334         if (qid->reorder_buffer_freelist) {
 335                 rte_ring_free(qid->reorder_buffer_freelist);
 336                 qid->reorder_buffer_freelist = NULL;
 337         }
 338
 339         return -EINVAL;
 340 }
 341
 342 static int
 343 sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
 344                 const struct rte_event_queue_conf *conf)
 345 {
 346         int type;
 347
 348         /* SINGLE_LINK can be OR-ed with other types, so handle first */
 349         if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) {
 350                 type = SW_SCHED_TYPE_DIRECT;
 351         } else {
 352                 switch (conf->event_queue_cfg) {
 353                 case RTE_EVENT_QUEUE_CFG_ATOMIC_ONLY:
 354                         type = RTE_SCHED_TYPE_ATOMIC;
 355                         break;
 356                 case RTE_EVENT_QUEUE_CFG_ORDERED_ONLY:
 357                         type = RTE_SCHED_TYPE_ORDERED;
 358                         break;
 359                 case RTE_EVENT_QUEUE_CFG_PARALLEL_ONLY:
 360                         type = RTE_SCHED_TYPE_PARALLEL;
 361                         break;
 362                 case RTE_EVENT_QUEUE_CFG_ALL_TYPES:
 363                         SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n");
 364                         return -ENOTSUP;
 365                 default:
 366                         SW_LOG_ERR("Unknown queue type %d requested\n",
 367                                    conf->event_queue_cfg);
 368                         return -EINVAL;
 369                 }
 370         }
 371
 372         struct sw_evdev *sw = sw_pmd_priv(dev);
 373         return qid_init(sw, queue_id, type, conf);
 374 }
 375
 376 static void
 377 sw_queue_release(struct rte_eventdev *dev, uint8_t id)
 378 {
 379         struct sw_evdev *sw = sw_pmd_priv(dev);
 380         struct sw_qid *qid = &sw->qids[id];
 381         uint32_t i;
 382
 383         for (i = 0; i < SW_IQS_MAX; i++)
 384                 iq_ring_destroy(qid->iq[i]);
 385
 386         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 387                 rte_free(qid->reorder_buffer);
 388                 rte_ring_free(qid->reorder_buffer_freelist);
 389         }
 390         memset(qid, 0, sizeof(*qid));
 391 }
 392
 393 static void
 394 sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
 395                                  struct rte_event_queue_conf *conf)
 396 {
 397         RTE_SET_USED(dev);
 398         RTE_SET_USED(queue_id);
 399
 400         static const struct rte_event_queue_conf default_conf = {
 401                 .nb_atomic_flows = 4096,
 402                 .nb_atomic_order_sequences = 1,
 403                 .event_queue_cfg = RTE_EVENT_QUEUE_CFG_ATOMIC_ONLY,
 404                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 405         };
 406
 407         *conf = default_conf;
 408 }
 409
 410 static void
 411 sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 412                  struct rte_event_port_conf *port_conf)
 413 {
 414         RTE_SET_USED(dev);
 415         RTE_SET_USED(port_id);
 416
 417         port_conf->new_event_threshold = 1024;
 418         port_conf->dequeue_depth = 16;
 419         port_conf->enqueue_depth = 16;
 420 }
 421
 422 static int
 423 sw_dev_configure(const struct rte_eventdev *dev)
 424 {
 425         struct sw_evdev *sw = sw_pmd_priv(dev);
 426         const struct rte_eventdev_data *data = dev->data;
 427         const struct rte_event_dev_config *conf = &data->dev_conf;
 428
 429         sw->qid_count = conf->nb_event_queues;
 430         sw->port_count = conf->nb_event_ports;
 431         sw->nb_events_limit = conf->nb_events_limit;
 432         rte_atomic32_set(&sw->inflights, 0);
 433
 434         if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
 435                 return -ENOTSUP;
 436
 437         return 0;
 438 }
 439
 440 static void
 441 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info)
 442 {
 443         RTE_SET_USED(dev);
 444
 445         static const struct rte_event_dev_info evdev_sw_info = {
 446                         .driver_name = SW_PMD_NAME,
 447                         .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
 448                         .max_event_queue_flows = SW_QID_NUM_FIDS,
 449                         .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX,
 450                         .max_event_priority_levels = SW_IQS_MAX,
 451                         .max_event_ports = SW_PORTS_MAX,
 452                         .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH,
 453                         .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH,
 454                         .max_num_events = SW_INFLIGHT_EVENTS_TOTAL,
 455                         .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
 456                                         RTE_EVENT_DEV_CAP_BURST_MODE |
 457                                         RTE_EVENT_DEV_CAP_EVENT_QOS),
 458         };
 459
 460         *info = evdev_sw_info;
 461 }
 462
 463 static void
 464 sw_dump(struct rte_eventdev *dev, FILE *f)
 465 {
 466         const struct sw_evdev *sw = sw_pmd_priv(dev);
 467
 468         static const char * const q_type_strings[] = {
 469                         "Ordered", "Atomic", "Parallel", "Directed"
 470         };
 471         uint32_t i;
 472         fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name",
 473                         sw->port_count, sw->qid_count);
 474
 475         fprintf(f, "\trx   %"PRIu64"\n\tdrop %"PRIu64"\n\ttx   %"PRIu64"\n",
 476                 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts);
 477         fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called);
 478         fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called);
 479         fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues);
 480         fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues);
 481         uint32_t inflights = rte_atomic32_read(&sw->inflights);
 482         uint32_t credits = sw->nb_events_limit - inflights;
 483         fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits);
 484
 485 #define COL_RED "\x1b[31m"
 486 #define COL_RESET "\x1b[0m"
 487
 488         for (i = 0; i < sw->port_count; i++) {
 489                 int max, j;
 490                 const struct sw_port *p = &sw->ports[i];
 491                 if (!p->initialized) {
 492                         fprintf(f, "  %sPort %d not initialized.%s\n",
 493                                 COL_RED, i, COL_RESET);
 494                         continue;
 495                 }
 496                 fprintf(f, "  Port %d %s\n", i,
 497                         p->is_directed ? " (SingleCons)" : "");
 498                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64
 499                         "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts,
 500                         sw->ports[i].stats.rx_dropped,
 501                         sw->ports[i].stats.tx_pkts,
 502                         (p->inflights == p->inflight_max) ?
 503                                 COL_RED : COL_RESET,
 504                         sw->ports[i].inflights, COL_RESET);
 505
 506                 fprintf(f, "\tMax New: %u"
 507                         "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n",
 508                         sw->ports[i].inflight_max,
 509                         sw->ports[i].avg_pkt_ticks,
 510                         sw->ports[i].inflight_credits);
 511                 fprintf(f, "\tReceive burst distribution:\n");
 512                 float zp_percent = p->zero_polls * 100.0 / p->total_polls;
 513                 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ",
 514                                 zp_percent);
 515                 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;)
 516                         if (p->poll_buckets[max] != 0)
 517                                 break;
 518                 for (j = 0; j <= max; j++) {
 519                         if (p->poll_buckets[j] != 0) {
 520                                 float poll_pc = p->poll_buckets[j] * 100.0 /
 521                                         p->total_polls;
 522                                 fprintf(f, "%u-%u:%.02f%% ",
 523                                         ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1),
 524                                         ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT),
 525                                         poll_pc);
 526                         }
 527                 }
 528                 fprintf(f, "\n");
 529
 530                 if (p->rx_worker_ring) {
 531                         uint64_t used = rte_event_ring_count(p->rx_worker_ring);
 532                         uint64_t space = rte_event_ring_free_count(
 533                                         p->rx_worker_ring);
 534                         const char *col = (space == 0) ? COL_RED : COL_RESET;
 535                         fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4"
 536                                         PRIu64 COL_RESET"\n", col, used, space);
 537                 } else
 538                         fprintf(f, "\trx ring not initialized.\n");
 539
 540                 if (p->cq_worker_ring) {
 541                         uint64_t used = rte_event_ring_count(p->cq_worker_ring);
 542                         uint64_t space = rte_event_ring_free_count(
 543                                         p->cq_worker_ring);
 544                         const char *col = (space == 0) ? COL_RED : COL_RESET;
 545                         fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4"
 546                                         PRIu64 COL_RESET"\n", col, used, space);
 547                 } else
 548                         fprintf(f, "\tcq ring not initialized.\n");
 549         }
 550
 551         for (i = 0; i < sw->qid_count; i++) {
 552                 const struct sw_qid *qid = &sw->qids[i];
 553                 if (!qid->initialized) {
 554                         fprintf(f, "  %sQueue %d not initialized.%s\n",
 555                                 COL_RED, i, COL_RESET);
 556                         continue;
 557                 }
 558                 int affinities_per_port[SW_PORTS_MAX] = {0};
 559                 uint32_t inflights = 0;
 560
 561                 fprintf(f, "  Queue %d (%s)\n", i, q_type_strings[qid->type]);
 562                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64"\n",
 563                         qid->stats.rx_pkts, qid->stats.rx_dropped,
 564                         qid->stats.tx_pkts);
 565                 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 566                         struct rte_ring *rob_buf_free =
 567                                 qid->reorder_buffer_freelist;
 568                         if (rob_buf_free)
 569                                 fprintf(f, "\tReorder entries in use: %u\n",
 570                                         rte_ring_free_count(rob_buf_free));
 571                         else
 572                                 fprintf(f,
 573                                         "\tReorder buffer not initialized\n");
 574                 }
 575
 576                 uint32_t flow;
 577                 for (flow = 0; flow < RTE_DIM(qid->fids); flow++)
 578                         if (qid->fids[flow].cq != -1) {
 579                                 affinities_per_port[qid->fids[flow].cq]++;
 580                                 inflights += qid->fids[flow].pcount;
 581                         }
 582
 583                 uint32_t port;
 584                 fprintf(f, "\tPer Port Stats:\n");
 585                 for (port = 0; port < sw->port_count; port++) {
 586                         fprintf(f, "\t  Port %d: Pkts: %"PRIu64, port,
 587                                         qid->to_port[port]);
 588                         fprintf(f, "\tFlows: %d\n", affinities_per_port[port]);
 589                 }
 590
 591                 uint32_t iq;
 592                 uint32_t iq_printed = 0;
 593                 for (iq = 0; iq < SW_IQS_MAX; iq++) {
 594                         if (!qid->iq[iq]) {
 595                                 fprintf(f, "\tiq %d is not initialized.\n", iq);
 596                                 iq_printed = 1;
 597                                 continue;
 598                         }
 599                         uint32_t used = iq_ring_count(qid->iq[iq]);
 600                         uint32_t free = iq_ring_free_count(qid->iq[iq]);
 601                         const char *col = (free == 0) ? COL_RED : COL_RESET;
 602                         if (used > 0) {
 603                                 fprintf(f, "\t%siq %d: Used %d\tFree %d"
 604                                         COL_RESET"\n", col, iq, used, free);
 605                                 iq_printed = 1;
 606                         }
 607                 }
 608                 if (iq_printed == 0)
 609                         fprintf(f, "\t-- iqs empty --\n");
 610         }
 611 }
 612
 613 static int
 614 sw_start(struct rte_eventdev *dev)
 615 {
 616         unsigned int i, j;
 617         struct sw_evdev *sw = sw_pmd_priv(dev);
 618
 619         /* check a service core is mapped to this service */
 620         struct rte_service_spec *s = rte_service_get_by_name(sw->service_name);
 621         if (!rte_service_is_running(s))
 622                 SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
 623                                 s->name);
 624
 625         /* check all ports are set up */
 626         for (i = 0; i < sw->port_count; i++)
 627                 if (sw->ports[i].rx_worker_ring == NULL) {
 628                         SW_LOG_ERR("Port %d not configured\n", i);
 629                         return -ESTALE;
 630                 }
 631
 632         /* check all queues are configured and mapped to ports*/
 633         for (i = 0; i < sw->qid_count; i++)
 634                 if (sw->qids[i].iq[0] == NULL ||
 635                                 sw->qids[i].cq_num_mapped_cqs == 0) {
 636                         SW_LOG_ERR("Queue %d not configured\n", i);
 637                         return -ENOLINK;
 638                 }
 639
 640         /* build up our prioritized array of qids */
 641         /* We don't use qsort here, as if all/multiple entries have the same
 642          * priority, the result is non-deterministic. From "man 3 qsort":
 643          * "If two members compare as equal, their order in the sorted
 644          * array is undefined."
 645          */
 646         uint32_t qidx = 0;
 647         for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) {
 648                 for (i = 0; i < sw->qid_count; i++) {
 649                         if (sw->qids[i].priority == j) {
 650                                 sw->qids_prioritized[qidx] = &sw->qids[i];
 651                                 qidx++;
 652                         }
 653                 }
 654         }
 655
 656         if (sw_xstats_init(sw) < 0)
 657                 return -EINVAL;
 658
 659         rte_smp_wmb();
 660         sw->started = 1;
 661
 662         return 0;
 663 }
 664
 665 static void
 666 sw_stop(struct rte_eventdev *dev)
 667 {
 668         struct sw_evdev *sw = sw_pmd_priv(dev);
 669         sw_xstats_uninit(sw);
 670         sw->started = 0;
 671         rte_smp_wmb();
 672 }
 673
 674 static int
 675 sw_close(struct rte_eventdev *dev)
 676 {
 677         struct sw_evdev *sw = sw_pmd_priv(dev);
 678         uint32_t i;
 679
 680         for (i = 0; i < sw->qid_count; i++)
 681                 sw_queue_release(dev, i);
 682         sw->qid_count = 0;
 683
 684         for (i = 0; i < sw->port_count; i++)
 685                 sw_port_release(&sw->ports[i]);
 686         sw->port_count = 0;
 687
 688         memset(&sw->stats, 0, sizeof(sw->stats));
 689         sw->sched_called = 0;
 690         sw->sched_no_iq_enqueues = 0;
 691         sw->sched_no_cq_enqueues = 0;
 692         sw->sched_cq_qid_called = 0;
 693
 694         return 0;
 695 }
 696
 697 static int
 698 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque)
 699 {
 700         int *socket_id = opaque;
 701         *socket_id = atoi(value);
 702         if (*socket_id >= RTE_MAX_NUMA_NODES)
 703                 return -1;
 704         return 0;
 705 }
 706
 707 static int
 708 set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque)
 709 {
 710         int *quanta = opaque;
 711         *quanta = atoi(value);
 712         if (*quanta < 0 || *quanta >= 4096)
 713                 return -1;
 714         return 0;
 715 }
 716
 717 static int
 718 set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
 719 {
 720         int *credit = opaque;
 721         *credit = atoi(value);
 722         if (*credit < 0 || *credit >= 128)
 723                 return -1;
 724         return 0;
 725 }
 726
 727
 728 static int32_t sw_sched_service_func(void *args)
 729 {
 730         struct rte_eventdev *dev = args;
 731         sw_event_schedule(dev);
 732         return 0;
 733 }
 734
 735 static int
 736 sw_probe(struct rte_vdev_device *vdev)
 737 {
 738         static const struct rte_eventdev_ops evdev_sw_ops = {
 739                         .dev_configure = sw_dev_configure,
 740                         .dev_infos_get = sw_info_get,
 741                         .dev_close = sw_close,
 742                         .dev_start = sw_start,
 743                         .dev_stop = sw_stop,
 744                         .dump = sw_dump,
 745
 746                         .queue_def_conf = sw_queue_def_conf,
 747                         .queue_setup = sw_queue_setup,
 748                         .queue_release = sw_queue_release,
 749                         .port_def_conf = sw_port_def_conf,
 750                         .port_setup = sw_port_setup,
 751                         .port_release = sw_port_release,
 752                         .port_link = sw_port_link,
 753                         .port_unlink = sw_port_unlink,
 754
 755                         .xstats_get = sw_xstats_get,
 756                         .xstats_get_names = sw_xstats_get_names,
 757                         .xstats_get_by_name = sw_xstats_get_by_name,
 758                         .xstats_reset = sw_xstats_reset,
 759         };
 760
 761         static const char *const args[] = {
 762                 NUMA_NODE_ARG,
 763                 SCHED_QUANTA_ARG,
 764                 CREDIT_QUANTA_ARG,
 765                 NULL
 766         };
 767         const char *name;
 768         const char *params;
 769         struct rte_eventdev *dev;
 770         struct sw_evdev *sw;
 771         int socket_id = rte_socket_id();
 772         int sched_quanta  = SW_DEFAULT_SCHED_QUANTA;
 773         int credit_quanta = SW_DEFAULT_CREDIT_QUANTA;
 774
 775         name = rte_vdev_device_name(vdev);
 776         params = rte_vdev_device_args(vdev);
 777         if (params != NULL && params[0] != '\0') {
 778                 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
 779
 780                 if (!kvlist) {
 781                         SW_LOG_INFO(
 782                                 "Ignoring unsupported parameters when creating device '%s'\n",
 783                                 name);
 784                 } else {
 785                         int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
 786                                         assign_numa_node, &socket_id);
 787                         if (ret != 0) {
 788                                 SW_LOG_ERR(
 789                                         "%s: Error parsing numa node parameter",
 790                                         name);
 791                                 rte_kvargs_free(kvlist);
 792                                 return ret;
 793                         }
 794
 795                         ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG,
 796                                         set_sched_quanta, &sched_quanta);
 797                         if (ret != 0) {
 798                                 SW_LOG_ERR(
 799                                         "%s: Error parsing sched quanta parameter",
 800                                         name);
 801                                 rte_kvargs_free(kvlist);
 802                                 return ret;
 803                         }
 804
 805                         ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG,
 806                                         set_credit_quanta, &credit_quanta);
 807                         if (ret != 0) {
 808                                 SW_LOG_ERR(
 809                                         "%s: Error parsing credit quanta parameter",
 810                                         name);
 811                                 rte_kvargs_free(kvlist);
 812                                 return ret;
 813                         }
 814
 815                         rte_kvargs_free(kvlist);
 816                 }
 817         }
 818
 819         SW_LOG_INFO(
 820                         "Creating eventdev sw device %s, numa_node=%d, sched_quanta=%d, credit_quanta=%d\n",
 821                         name, socket_id, sched_quanta, credit_quanta);
 822
 823         dev = rte_event_pmd_vdev_init(name,
 824                         sizeof(struct sw_evdev), socket_id);
 825         if (dev == NULL) {
 826                 SW_LOG_ERR("eventdev vdev init() failed");
 827                 return -EFAULT;
 828         }
 829         dev->dev_ops = &evdev_sw_ops;
 830         dev->enqueue = sw_event_enqueue;
 831         dev->enqueue_burst = sw_event_enqueue_burst;
 832         dev->enqueue_new_burst = sw_event_enqueue_burst;
 833         dev->enqueue_forward_burst = sw_event_enqueue_burst;
 834         dev->dequeue = sw_event_dequeue;
 835         dev->dequeue_burst = sw_event_dequeue_burst;
 836         dev->schedule = sw_event_schedule;
 837
 838         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 839                 return 0;
 840
 841         sw = dev->data->dev_private;
 842         sw->data = dev->data;
 843
 844         /* copy values passed from vdev command line to instance */
 845         sw->credit_update_quanta = credit_quanta;
 846         sw->sched_quanta = sched_quanta;
 847
 848         /* register service with EAL */
 849         struct rte_service_spec service;
 850         memset(&service, 0, sizeof(struct rte_service_spec));
 851         snprintf(service.name, sizeof(service.name), "%s_service", name);
 852         snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
 853                         name);
 854         service.socket_id = socket_id;
 855         service.callback = sw_sched_service_func;
 856         service.callback_userdata = (void *)dev;
 857
 858         int32_t ret = rte_service_register(&service);
 859         if (ret) {
 860                 SW_LOG_ERR("service register() failed");
 861                 return -ENOEXEC;
 862         }
 863
 864         return 0;
 865 }
 866
 867 static int
 868 sw_remove(struct rte_vdev_device *vdev)
 869 {
 870         const char *name;
 871
 872         name = rte_vdev_device_name(vdev);
 873         if (name == NULL)
 874                 return -EINVAL;
 875
 876         SW_LOG_INFO("Closing eventdev sw device %s\n", name);
 877
 878         return rte_event_pmd_vdev_uninit(name);
 879 }
 880
 881 static struct rte_vdev_driver evdev_sw_pmd_drv = {
 882         .probe = sw_probe,
 883         .remove = sw_remove
 884 };
 885
 886 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv);
 887 RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> "
 888                 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>");