drivers/event/sw/sw_evdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2016-2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <inttypes.h>
  34 #include <string.h>
  35
  36 #include <rte_bus_vdev.h>
  37 #include <rte_kvargs.h>
  38 #include <rte_ring.h>
  39 #include <rte_errno.h>
  40 #include <rte_event_ring.h>
  41 #include <rte_service_component.h>
  42
  43 #include "sw_evdev.h"
  44 #include "iq_ring.h"
  45
  46 #define EVENTDEV_NAME_SW_PMD event_sw
  47 #define NUMA_NODE_ARG "numa_node"
  48 #define SCHED_QUANTA_ARG "sched_quanta"
  49 #define CREDIT_QUANTA_ARG "credit_quanta"
  50
  51 static void
  52 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info);
  53
  54 static int
  55 sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[],
  56                 const uint8_t priorities[], uint16_t num)
  57 {
  58         struct sw_port *p = port;
  59         struct sw_evdev *sw = sw_pmd_priv(dev);
  60         int i;
  61
  62         RTE_SET_USED(priorities);
  63         for (i = 0; i < num; i++) {
  64                 struct sw_qid *q = &sw->qids[queues[i]];
  65                 unsigned int j;
  66
  67                 /* check for qid map overflow */
  68                 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) {
  69                         rte_errno = -EDQUOT;
  70                         break;
  71                 }
  72
  73                 if (p->is_directed && p->num_qids_mapped > 0) {
  74                         rte_errno = -EDQUOT;
  75                         break;
  76                 }
  77
  78                 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
  79                         if (q->cq_map[j] == p->id)
  80                                 break;
  81                 }
  82
  83                 /* check if port is already linked */
  84                 if (j < q->cq_num_mapped_cqs)
  85                         continue;
  86
  87                 if (q->type == SW_SCHED_TYPE_DIRECT) {
  88                         /* check directed qids only map to one port */
  89                         if (p->num_qids_mapped > 0) {
  90                                 rte_errno = -EDQUOT;
  91                                 break;
  92                         }
  93                         /* check port only takes a directed flow */
  94                         if (num > 1) {
  95                                 rte_errno = -EDQUOT;
  96                                 break;
  97                         }
  98
  99                         p->is_directed = 1;
 100                         p->num_qids_mapped = 1;
 101                 } else if (q->type == RTE_SCHED_TYPE_ORDERED) {
 102                         p->num_ordered_qids++;
 103                         p->num_qids_mapped++;
 104                 } else if (q->type == RTE_SCHED_TYPE_ATOMIC ||
 105                                 q->type == RTE_SCHED_TYPE_PARALLEL) {
 106                         p->num_qids_mapped++;
 107                 }
 108
 109                 q->cq_map[q->cq_num_mapped_cqs] = p->id;
 110                 rte_smp_wmb();
 111                 q->cq_num_mapped_cqs++;
 112         }
 113         return i;
 114 }
 115
 116 static int
 117 sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[],
 118                 uint16_t nb_unlinks)
 119 {
 120         struct sw_port *p = port;
 121         struct sw_evdev *sw = sw_pmd_priv(dev);
 122         unsigned int i, j;
 123
 124         int unlinked = 0;
 125         for (i = 0; i < nb_unlinks; i++) {
 126                 struct sw_qid *q = &sw->qids[queues[i]];
 127                 for (j = 0; j < q->cq_num_mapped_cqs; j++) {
 128                         if (q->cq_map[j] == p->id) {
 129                                 q->cq_map[j] =
 130                                         q->cq_map[q->cq_num_mapped_cqs - 1];
 131                                 rte_smp_wmb();
 132                                 q->cq_num_mapped_cqs--;
 133                                 unlinked++;
 134
 135                                 p->num_qids_mapped--;
 136
 137                                 if (q->type == RTE_SCHED_TYPE_ORDERED)
 138                                         p->num_ordered_qids--;
 139
 140                                 continue;
 141                         }
 142                 }
 143         }
 144         return unlinked;
 145 }
 146
 147 static int
 148 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id,
 149                 const struct rte_event_port_conf *conf)
 150 {
 151         struct sw_evdev *sw = sw_pmd_priv(dev);
 152         struct sw_port *p = &sw->ports[port_id];
 153         char buf[RTE_RING_NAMESIZE];
 154         unsigned int i;
 155
 156         struct rte_event_dev_info info;
 157         sw_info_get(dev, &info);
 158
 159         /* detect re-configuring and return credits to instance if needed */
 160         if (p->initialized) {
 161                 /* taking credits from pool is done one quanta at a time, and
 162                  * credits may be spend (counted in p->inflights) or still
 163                  * available in the port (p->inflight_credits). We must return
 164                  * the sum to no leak credits
 165                  */
 166                 int possible_inflights = p->inflight_credits + p->inflights;
 167                 rte_atomic32_sub(&sw->inflights, possible_inflights);
 168         }
 169
 170         *p = (struct sw_port){0}; /* zero entire structure */
 171         p->id = port_id;
 172         p->sw = sw;
 173
 174         /* check to see if rings exists - port_setup() can be called multiple
 175          * times legally (assuming device is stopped). If ring exists, free it
 176          * to so it gets re-created with the correct size
 177          */
 178         snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id,
 179                         port_id, "rx_worker_ring");
 180         struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf);
 181         if (existing_ring)
 182                 rte_event_ring_free(existing_ring);
 183
 184         p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH,
 185                         dev->data->socket_id,
 186                         RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
 187         if (p->rx_worker_ring == NULL) {
 188                 SW_LOG_ERR("Error creating RX worker ring for port %d\n",
 189                                 port_id);
 190                 return -1;
 191         }
 192
 193         p->inflight_max = conf->new_event_threshold;
 194
 195         /* check if ring exists, same as rx_worker above */
 196         snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id,
 197                         port_id, "cq_worker_ring");
 198         existing_ring = rte_event_ring_lookup(buf);
 199         if (existing_ring)
 200                 rte_event_ring_free(existing_ring);
 201
 202         p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth,
 203                         dev->data->socket_id,
 204                         RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
 205         if (p->cq_worker_ring == NULL) {
 206                 rte_event_ring_free(p->rx_worker_ring);
 207                 SW_LOG_ERR("Error creating CQ worker ring for port %d\n",
 208                                 port_id);
 209                 return -1;
 210         }
 211         sw->cq_ring_space[port_id] = conf->dequeue_depth;
 212
 213         /* set hist list contents to empty */
 214         for (i = 0; i < SW_PORT_HIST_LIST; i++) {
 215                 p->hist_list[i].fid = -1;
 216                 p->hist_list[i].qid = -1;
 217         }
 218         dev->data->ports[port_id] = p;
 219
 220         rte_smp_wmb();
 221         p->initialized = 1;
 222         return 0;
 223 }
 224
 225 static void
 226 sw_port_release(void *port)
 227 {
 228         struct sw_port *p = (void *)port;
 229         if (p == NULL)
 230                 return;
 231
 232         rte_event_ring_free(p->rx_worker_ring);
 233         rte_event_ring_free(p->cq_worker_ring);
 234         memset(p, 0, sizeof(*p));
 235 }
 236
 237 static int32_t
 238 qid_init(struct sw_evdev *sw, unsigned int idx, int type,
 239                 const struct rte_event_queue_conf *queue_conf)
 240 {
 241         unsigned int i;
 242         int dev_id = sw->data->dev_id;
 243         int socket_id = sw->data->socket_id;
 244         char buf[IQ_RING_NAMESIZE];
 245         struct sw_qid *qid = &sw->qids[idx];
 246
 247         for (i = 0; i < SW_IQS_MAX; i++) {
 248                 snprintf(buf, sizeof(buf), "q_%u_iq_%d", idx, i);
 249                 qid->iq[i] = iq_ring_create(buf, socket_id);
 250                 if (!qid->iq[i]) {
 251                         SW_LOG_DBG("ring create failed");
 252                         goto cleanup;
 253                 }
 254         }
 255
 256         /* Initialize the FID structures to no pinning (-1), and zero packets */
 257         const struct sw_fid_t fid = {.cq = -1, .pcount = 0};
 258         for (i = 0; i < RTE_DIM(qid->fids); i++)
 259                 qid->fids[i] = fid;
 260
 261         qid->id = idx;
 262         qid->type = type;
 263         qid->priority = queue_conf->priority;
 264
 265         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 266                 char ring_name[RTE_RING_NAMESIZE];
 267                 uint32_t window_size;
 268
 269                 /* rte_ring and window_size_mask require require window_size to
 270                  * be a power-of-2.
 271                  */
 272                 window_size = rte_align32pow2(
 273                                 queue_conf->nb_atomic_order_sequences);
 274
 275                 qid->window_size = window_size - 1;
 276
 277                 if (!window_size) {
 278                         SW_LOG_DBG(
 279                                 "invalid reorder_window_size for ordered queue\n"
 280                                 );
 281                         goto cleanup;
 282                 }
 283
 284                 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i);
 285                 qid->reorder_buffer = rte_zmalloc_socket(buf,
 286                                 window_size * sizeof(qid->reorder_buffer[0]),
 287                                 0, socket_id);
 288                 if (!qid->reorder_buffer) {
 289                         SW_LOG_DBG("reorder_buffer malloc failed\n");
 290                         goto cleanup;
 291                 }
 292
 293                 memset(&qid->reorder_buffer[0],
 294                        0,
 295                        window_size * sizeof(qid->reorder_buffer[0]));
 296
 297                 snprintf(ring_name, sizeof(ring_name), "sw%d_q%d_freelist",
 298                                 dev_id, idx);
 299
 300                 /* lookup the ring, and if it already exists, free it */
 301                 struct rte_ring *cleanup = rte_ring_lookup(ring_name);
 302                 if (cleanup)
 303                         rte_ring_free(cleanup);
 304
 305                 qid->reorder_buffer_freelist = rte_ring_create(ring_name,
 306                                 window_size,
 307                                 socket_id,
 308                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
 309                 if (!qid->reorder_buffer_freelist) {
 310                         SW_LOG_DBG("freelist ring create failed");
 311                         goto cleanup;
 312                 }
 313
 314                 /* Populate the freelist with reorder buffer entries. Enqueue
 315                  * 'window_size - 1' entries because the rte_ring holds only
 316                  * that many.
 317                  */
 318                 for (i = 0; i < window_size - 1; i++) {
 319                         if (rte_ring_sp_enqueue(qid->reorder_buffer_freelist,
 320                                                 &qid->reorder_buffer[i]) < 0)
 321                                 goto cleanup;
 322                 }
 323
 324                 qid->reorder_buffer_index = 0;
 325                 qid->cq_next_tx = 0;
 326         }
 327
 328         qid->initialized = 1;
 329
 330         return 0;
 331
 332 cleanup:
 333         for (i = 0; i < SW_IQS_MAX; i++) {
 334                 if (qid->iq[i])
 335                         iq_ring_destroy(qid->iq[i]);
 336         }
 337
 338         if (qid->reorder_buffer) {
 339                 rte_free(qid->reorder_buffer);
 340                 qid->reorder_buffer = NULL;
 341         }
 342
 343         if (qid->reorder_buffer_freelist) {
 344                 rte_ring_free(qid->reorder_buffer_freelist);
 345                 qid->reorder_buffer_freelist = NULL;
 346         }
 347
 348         return -EINVAL;
 349 }
 350
 351 static void
 352 sw_queue_release(struct rte_eventdev *dev, uint8_t id)
 353 {
 354         struct sw_evdev *sw = sw_pmd_priv(dev);
 355         struct sw_qid *qid = &sw->qids[id];
 356         uint32_t i;
 357
 358         for (i = 0; i < SW_IQS_MAX; i++)
 359                 iq_ring_destroy(qid->iq[i]);
 360
 361         if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 362                 rte_free(qid->reorder_buffer);
 363                 rte_ring_free(qid->reorder_buffer_freelist);
 364         }
 365         memset(qid, 0, sizeof(*qid));
 366 }
 367
 368 static int
 369 sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
 370                 const struct rte_event_queue_conf *conf)
 371 {
 372         int type;
 373
 374         type = conf->schedule_type;
 375
 376         if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) {
 377                 type = SW_SCHED_TYPE_DIRECT;
 378         } else if (RTE_EVENT_QUEUE_CFG_ALL_TYPES
 379                         & conf->event_queue_cfg) {
 380                 SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n");
 381                 return -ENOTSUP;
 382         }
 383
 384         struct sw_evdev *sw = sw_pmd_priv(dev);
 385
 386         if (sw->qids[queue_id].initialized)
 387                 sw_queue_release(dev, queue_id);
 388
 389         return qid_init(sw, queue_id, type, conf);
 390 }
 391
 392 static void
 393 sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
 394                                  struct rte_event_queue_conf *conf)
 395 {
 396         RTE_SET_USED(dev);
 397         RTE_SET_USED(queue_id);
 398
 399         static const struct rte_event_queue_conf default_conf = {
 400                 .nb_atomic_flows = 4096,
 401                 .nb_atomic_order_sequences = 1,
 402                 .schedule_type = RTE_SCHED_TYPE_ATOMIC,
 403                 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
 404         };
 405
 406         *conf = default_conf;
 407 }
 408
 409 static void
 410 sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
 411                  struct rte_event_port_conf *port_conf)
 412 {
 413         RTE_SET_USED(dev);
 414         RTE_SET_USED(port_id);
 415
 416         port_conf->new_event_threshold = 1024;
 417         port_conf->dequeue_depth = 16;
 418         port_conf->enqueue_depth = 16;
 419 }
 420
 421 static int
 422 sw_dev_configure(const struct rte_eventdev *dev)
 423 {
 424         struct sw_evdev *sw = sw_pmd_priv(dev);
 425         const struct rte_eventdev_data *data = dev->data;
 426         const struct rte_event_dev_config *conf = &data->dev_conf;
 427
 428         sw->qid_count = conf->nb_event_queues;
 429         sw->port_count = conf->nb_event_ports;
 430         sw->nb_events_limit = conf->nb_events_limit;
 431         rte_atomic32_set(&sw->inflights, 0);
 432
 433         if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT)
 434                 return -ENOTSUP;
 435
 436         return 0;
 437 }
 438
 439 struct rte_eth_dev;
 440
 441 static int
 442 sw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev,
 443                         const struct rte_eth_dev *eth_dev,
 444                         uint32_t *caps)
 445 {
 446         RTE_SET_USED(dev);
 447         RTE_SET_USED(eth_dev);
 448         *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP;
 449         return 0;
 450 }
 451
 452 static void
 453 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info)
 454 {
 455         RTE_SET_USED(dev);
 456
 457         static const struct rte_event_dev_info evdev_sw_info = {
 458                         .driver_name = SW_PMD_NAME,
 459                         .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
 460                         .max_event_queue_flows = SW_QID_NUM_FIDS,
 461                         .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX,
 462                         .max_event_priority_levels = SW_IQS_MAX,
 463                         .max_event_ports = SW_PORTS_MAX,
 464                         .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH,
 465                         .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH,
 466                         .max_num_events = SW_INFLIGHT_EVENTS_TOTAL,
 467                         .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
 468                                         RTE_EVENT_DEV_CAP_BURST_MODE |
 469                                         RTE_EVENT_DEV_CAP_EVENT_QOS),
 470         };
 471
 472         *info = evdev_sw_info;
 473 }
 474
 475 static void
 476 sw_dump(struct rte_eventdev *dev, FILE *f)
 477 {
 478         const struct sw_evdev *sw = sw_pmd_priv(dev);
 479
 480         static const char * const q_type_strings[] = {
 481                         "Ordered", "Atomic", "Parallel", "Directed"
 482         };
 483         uint32_t i;
 484         fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name",
 485                         sw->port_count, sw->qid_count);
 486
 487         fprintf(f, "\trx   %"PRIu64"\n\tdrop %"PRIu64"\n\ttx   %"PRIu64"\n",
 488                 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts);
 489         fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called);
 490         fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called);
 491         fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues);
 492         fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues);
 493         uint32_t inflights = rte_atomic32_read(&sw->inflights);
 494         uint32_t credits = sw->nb_events_limit - inflights;
 495         fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits);
 496
 497 #define COL_RED "\x1b[31m"
 498 #define COL_RESET "\x1b[0m"
 499
 500         for (i = 0; i < sw->port_count; i++) {
 501                 int max, j;
 502                 const struct sw_port *p = &sw->ports[i];
 503                 if (!p->initialized) {
 504                         fprintf(f, "  %sPort %d not initialized.%s\n",
 505                                 COL_RED, i, COL_RESET);
 506                         continue;
 507                 }
 508                 fprintf(f, "  Port %d %s\n", i,
 509                         p->is_directed ? " (SingleCons)" : "");
 510                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64
 511                         "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts,
 512                         sw->ports[i].stats.rx_dropped,
 513                         sw->ports[i].stats.tx_pkts,
 514                         (p->inflights == p->inflight_max) ?
 515                                 COL_RED : COL_RESET,
 516                         sw->ports[i].inflights, COL_RESET);
 517
 518                 fprintf(f, "\tMax New: %u"
 519                         "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n",
 520                         sw->ports[i].inflight_max,
 521                         sw->ports[i].avg_pkt_ticks,
 522                         sw->ports[i].inflight_credits);
 523                 fprintf(f, "\tReceive burst distribution:\n");
 524                 float zp_percent = p->zero_polls * 100.0 / p->total_polls;
 525                 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ",
 526                                 zp_percent);
 527                 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;)
 528                         if (p->poll_buckets[max] != 0)
 529                                 break;
 530                 for (j = 0; j <= max; j++) {
 531                         if (p->poll_buckets[j] != 0) {
 532                                 float poll_pc = p->poll_buckets[j] * 100.0 /
 533                                         p->total_polls;
 534                                 fprintf(f, "%u-%u:%.02f%% ",
 535                                         ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1),
 536                                         ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT),
 537                                         poll_pc);
 538                         }
 539                 }
 540                 fprintf(f, "\n");
 541
 542                 if (p->rx_worker_ring) {
 543                         uint64_t used = rte_event_ring_count(p->rx_worker_ring);
 544                         uint64_t space = rte_event_ring_free_count(
 545                                         p->rx_worker_ring);
 546                         const char *col = (space == 0) ? COL_RED : COL_RESET;
 547                         fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4"
 548                                         PRIu64 COL_RESET"\n", col, used, space);
 549                 } else
 550                         fprintf(f, "\trx ring not initialized.\n");
 551
 552                 if (p->cq_worker_ring) {
 553                         uint64_t used = rte_event_ring_count(p->cq_worker_ring);
 554                         uint64_t space = rte_event_ring_free_count(
 555                                         p->cq_worker_ring);
 556                         const char *col = (space == 0) ? COL_RED : COL_RESET;
 557                         fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4"
 558                                         PRIu64 COL_RESET"\n", col, used, space);
 559                 } else
 560                         fprintf(f, "\tcq ring not initialized.\n");
 561         }
 562
 563         for (i = 0; i < sw->qid_count; i++) {
 564                 const struct sw_qid *qid = &sw->qids[i];
 565                 if (!qid->initialized) {
 566                         fprintf(f, "  %sQueue %d not initialized.%s\n",
 567                                 COL_RED, i, COL_RESET);
 568                         continue;
 569                 }
 570                 int affinities_per_port[SW_PORTS_MAX] = {0};
 571                 uint32_t inflights = 0;
 572
 573                 fprintf(f, "  Queue %d (%s)\n", i, q_type_strings[qid->type]);
 574                 fprintf(f, "\trx   %"PRIu64"\tdrop %"PRIu64"\ttx   %"PRIu64"\n",
 575                         qid->stats.rx_pkts, qid->stats.rx_dropped,
 576                         qid->stats.tx_pkts);
 577                 if (qid->type == RTE_SCHED_TYPE_ORDERED) {
 578                         struct rte_ring *rob_buf_free =
 579                                 qid->reorder_buffer_freelist;
 580                         if (rob_buf_free)
 581                                 fprintf(f, "\tReorder entries in use: %u\n",
 582                                         rte_ring_free_count(rob_buf_free));
 583                         else
 584                                 fprintf(f,
 585                                         "\tReorder buffer not initialized\n");
 586                 }
 587
 588                 uint32_t flow;
 589                 for (flow = 0; flow < RTE_DIM(qid->fids); flow++)
 590                         if (qid->fids[flow].cq != -1) {
 591                                 affinities_per_port[qid->fids[flow].cq]++;
 592                                 inflights += qid->fids[flow].pcount;
 593                         }
 594
 595                 uint32_t port;
 596                 fprintf(f, "\tPer Port Stats:\n");
 597                 for (port = 0; port < sw->port_count; port++) {
 598                         fprintf(f, "\t  Port %d: Pkts: %"PRIu64, port,
 599                                         qid->to_port[port]);
 600                         fprintf(f, "\tFlows: %d\n", affinities_per_port[port]);
 601                 }
 602
 603                 uint32_t iq;
 604                 uint32_t iq_printed = 0;
 605                 for (iq = 0; iq < SW_IQS_MAX; iq++) {
 606                         if (!qid->iq[iq]) {
 607                                 fprintf(f, "\tiq %d is not initialized.\n", iq);
 608                                 iq_printed = 1;
 609                                 continue;
 610                         }
 611                         uint32_t used = iq_ring_count(qid->iq[iq]);
 612                         uint32_t free = iq_ring_free_count(qid->iq[iq]);
 613                         const char *col = (free == 0) ? COL_RED : COL_RESET;
 614                         if (used > 0) {
 615                                 fprintf(f, "\t%siq %d: Used %d\tFree %d"
 616                                         COL_RESET"\n", col, iq, used, free);
 617                                 iq_printed = 1;
 618                         }
 619                 }
 620                 if (iq_printed == 0)
 621                         fprintf(f, "\t-- iqs empty --\n");
 622         }
 623 }
 624
 625 static int
 626 sw_start(struct rte_eventdev *dev)
 627 {
 628         unsigned int i, j;
 629         struct sw_evdev *sw = sw_pmd_priv(dev);
 630
 631         rte_service_component_runstate_set(sw->service_id, 1);
 632
 633         /* check a service core is mapped to this service */
 634         if (!rte_service_runstate_get(sw->service_id)) {
 635                 SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
 636                                 sw->service_name);
 637                 return -ENOENT;
 638         }
 639
 640         /* check all ports are set up */
 641         for (i = 0; i < sw->port_count; i++)
 642                 if (sw->ports[i].rx_worker_ring == NULL) {
 643                         SW_LOG_ERR("Port %d not configured\n", i);
 644                         return -ESTALE;
 645                 }
 646
 647         /* check all queues are configured and mapped to ports*/
 648         for (i = 0; i < sw->qid_count; i++)
 649                 if (sw->qids[i].iq[0] == NULL ||
 650                                 sw->qids[i].cq_num_mapped_cqs == 0) {
 651                         SW_LOG_ERR("Queue %d not configured\n", i);
 652                         return -ENOLINK;
 653                 }
 654
 655         /* build up our prioritized array of qids */
 656         /* We don't use qsort here, as if all/multiple entries have the same
 657          * priority, the result is non-deterministic. From "man 3 qsort":
 658          * "If two members compare as equal, their order in the sorted
 659          * array is undefined."
 660          */
 661         uint32_t qidx = 0;
 662         for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) {
 663                 for (i = 0; i < sw->qid_count; i++) {
 664                         if (sw->qids[i].priority == j) {
 665                                 sw->qids_prioritized[qidx] = &sw->qids[i];
 666                                 qidx++;
 667                         }
 668                 }
 669         }
 670
 671         if (sw_xstats_init(sw) < 0)
 672                 return -EINVAL;
 673
 674         rte_smp_wmb();
 675         sw->started = 1;
 676
 677         return 0;
 678 }
 679
 680 static void
 681 sw_stop(struct rte_eventdev *dev)
 682 {
 683         struct sw_evdev *sw = sw_pmd_priv(dev);
 684         sw_xstats_uninit(sw);
 685         sw->started = 0;
 686         rte_smp_wmb();
 687 }
 688
 689 static int
 690 sw_close(struct rte_eventdev *dev)
 691 {
 692         struct sw_evdev *sw = sw_pmd_priv(dev);
 693         uint32_t i;
 694
 695         for (i = 0; i < sw->qid_count; i++)
 696                 sw_queue_release(dev, i);
 697         sw->qid_count = 0;
 698
 699         for (i = 0; i < sw->port_count; i++)
 700                 sw_port_release(&sw->ports[i]);
 701         sw->port_count = 0;
 702
 703         memset(&sw->stats, 0, sizeof(sw->stats));
 704         sw->sched_called = 0;
 705         sw->sched_no_iq_enqueues = 0;
 706         sw->sched_no_cq_enqueues = 0;
 707         sw->sched_cq_qid_called = 0;
 708
 709         return 0;
 710 }
 711
 712 static int
 713 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque)
 714 {
 715         int *socket_id = opaque;
 716         *socket_id = atoi(value);
 717         if (*socket_id >= RTE_MAX_NUMA_NODES)
 718                 return -1;
 719         return 0;
 720 }
 721
 722 static int
 723 set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque)
 724 {
 725         int *quanta = opaque;
 726         *quanta = atoi(value);
 727         if (*quanta < 0 || *quanta >= 4096)
 728                 return -1;
 729         return 0;
 730 }
 731
 732 static int
 733 set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
 734 {
 735         int *credit = opaque;
 736         *credit = atoi(value);
 737         if (*credit < 0 || *credit >= 128)
 738                 return -1;
 739         return 0;
 740 }
 741
 742
 743 static int32_t sw_sched_service_func(void *args)
 744 {
 745         struct rte_eventdev *dev = args;
 746         sw_event_schedule(dev);
 747         return 0;
 748 }
 749
 750 static int
 751 sw_probe(struct rte_vdev_device *vdev)
 752 {
 753         static const struct rte_eventdev_ops evdev_sw_ops = {
 754                         .dev_configure = sw_dev_configure,
 755                         .dev_infos_get = sw_info_get,
 756                         .dev_close = sw_close,
 757                         .dev_start = sw_start,
 758                         .dev_stop = sw_stop,
 759                         .dump = sw_dump,
 760
 761                         .queue_def_conf = sw_queue_def_conf,
 762                         .queue_setup = sw_queue_setup,
 763                         .queue_release = sw_queue_release,
 764                         .port_def_conf = sw_port_def_conf,
 765                         .port_setup = sw_port_setup,
 766                         .port_release = sw_port_release,
 767                         .port_link = sw_port_link,
 768                         .port_unlink = sw_port_unlink,
 769
 770                         .eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get,
 771
 772                         .xstats_get = sw_xstats_get,
 773                         .xstats_get_names = sw_xstats_get_names,
 774                         .xstats_get_by_name = sw_xstats_get_by_name,
 775                         .xstats_reset = sw_xstats_reset,
 776         };
 777
 778         static const char *const args[] = {
 779                 NUMA_NODE_ARG,
 780                 SCHED_QUANTA_ARG,
 781                 CREDIT_QUANTA_ARG,
 782                 NULL
 783         };
 784         const char *name;
 785         const char *params;
 786         struct rte_eventdev *dev;
 787         struct sw_evdev *sw;
 788         int socket_id = rte_socket_id();
 789         int sched_quanta  = SW_DEFAULT_SCHED_QUANTA;
 790         int credit_quanta = SW_DEFAULT_CREDIT_QUANTA;
 791
 792         name = rte_vdev_device_name(vdev);
 793         params = rte_vdev_device_args(vdev);
 794         if (params != NULL && params[0] != '\0') {
 795                 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
 796
 797                 if (!kvlist) {
 798                         SW_LOG_INFO(
 799                                 "Ignoring unsupported parameters when creating device '%s'\n",
 800                                 name);
 801                 } else {
 802                         int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
 803                                         assign_numa_node, &socket_id);
 804                         if (ret != 0) {
 805                                 SW_LOG_ERR(
 806                                         "%s: Error parsing numa node parameter",
 807                                         name);
 808                                 rte_kvargs_free(kvlist);
 809                                 return ret;
 810                         }
 811
 812                         ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG,
 813                                         set_sched_quanta, &sched_quanta);
 814                         if (ret != 0) {
 815                                 SW_LOG_ERR(
 816                                         "%s: Error parsing sched quanta parameter",
 817                                         name);
 818                                 rte_kvargs_free(kvlist);
 819                                 return ret;
 820                         }
 821
 822                         ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG,
 823                                         set_credit_quanta, &credit_quanta);
 824                         if (ret != 0) {
 825                                 SW_LOG_ERR(
 826                                         "%s: Error parsing credit quanta parameter",
 827                                         name);
 828                                 rte_kvargs_free(kvlist);
 829                                 return ret;
 830                         }
 831
 832                         rte_kvargs_free(kvlist);
 833                 }
 834         }
 835
 836         SW_LOG_INFO(
 837                         "Creating eventdev sw device %s, numa_node=%d, sched_quanta=%d, credit_quanta=%d\n",
 838                         name, socket_id, sched_quanta, credit_quanta);
 839
 840         dev = rte_event_pmd_vdev_init(name,
 841                         sizeof(struct sw_evdev), socket_id);
 842         if (dev == NULL) {
 843                 SW_LOG_ERR("eventdev vdev init() failed");
 844                 return -EFAULT;
 845         }
 846         dev->dev_ops = &evdev_sw_ops;
 847         dev->enqueue = sw_event_enqueue;
 848         dev->enqueue_burst = sw_event_enqueue_burst;
 849         dev->enqueue_new_burst = sw_event_enqueue_burst;
 850         dev->enqueue_forward_burst = sw_event_enqueue_burst;
 851         dev->dequeue = sw_event_dequeue;
 852         dev->dequeue_burst = sw_event_dequeue_burst;
 853
 854         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 855                 return 0;
 856
 857         sw = dev->data->dev_private;
 858         sw->data = dev->data;
 859
 860         /* copy values passed from vdev command line to instance */
 861         sw->credit_update_quanta = credit_quanta;
 862         sw->sched_quanta = sched_quanta;
 863
 864         /* register service with EAL */
 865         struct rte_service_spec service;
 866         memset(&service, 0, sizeof(struct rte_service_spec));
 867         snprintf(service.name, sizeof(service.name), "%s_service", name);
 868         snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
 869                         name);
 870         service.socket_id = socket_id;
 871         service.callback = sw_sched_service_func;
 872         service.callback_userdata = (void *)dev;
 873
 874         int32_t ret = rte_service_component_register(&service, &sw->service_id);
 875         if (ret) {
 876                 SW_LOG_ERR("service register() failed");
 877                 return -ENOEXEC;
 878         }
 879
 880         dev->data->service_inited = 1;
 881         dev->data->service_id = sw->service_id;
 882
 883         return 0;
 884 }
 885
 886 static int
 887 sw_remove(struct rte_vdev_device *vdev)
 888 {
 889         const char *name;
 890
 891         name = rte_vdev_device_name(vdev);
 892         if (name == NULL)
 893                 return -EINVAL;
 894
 895         SW_LOG_INFO("Closing eventdev sw device %s\n", name);
 896
 897         return rte_event_pmd_vdev_uninit(name);
 898 }
 899
 900 static struct rte_vdev_driver evdev_sw_pmd_drv = {
 901         .probe = sw_probe,
 902         .remove = sw_remove
 903 };
 904
 905 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv);
 906 RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> "
 907                 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>");