lib/librte_bbdev/rte_bbdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_compat.h>
  10 #include <rte_common.h>
  11 #include <rte_errno.h>
  12 #include <rte_log.h>
  13 #include <rte_debug.h>
  14 #include <rte_eal.h>
  15 #include <rte_malloc.h>
  16 #include <rte_mempool.h>
  17 #include <rte_memzone.h>
  18 #include <rte_lcore.h>
  19 #include <rte_dev.h>
  20 #include <rte_spinlock.h>
  21 #include <rte_tailq.h>
  22 #include <rte_interrupts.h>
  23
  24 #include "rte_bbdev_op.h"
  25 #include "rte_bbdev.h"
  26 #include "rte_bbdev_pmd.h"
  27
  28 #define DEV_NAME "BBDEV"
  29
  30
  31 /* BBDev library logging ID */
  32 static int bbdev_logtype;
  33
  34 /* Helper macro for logging */
  35 #define rte_bbdev_log(level, fmt, ...) \
  36         rte_log(RTE_LOG_ ## level, bbdev_logtype, fmt "\n", ##__VA_ARGS__)
  37
  38 #define rte_bbdev_log_debug(fmt, ...) \
  39         rte_bbdev_log(DEBUG, RTE_STR(__LINE__) ":%s() " fmt, __func__, \
  40                 ##__VA_ARGS__)
  41
  42 /* Helper macro to check dev_id is valid */
  43 #define VALID_DEV_OR_RET_ERR(dev, dev_id) do { \
  44         if (dev == NULL) { \
  45                 rte_bbdev_log(ERR, "device %u is invalid", dev_id); \
  46                 return -ENODEV; \
  47         } \
  48 } while (0)
  49
  50 /* Helper macro to check dev_ops is valid */
  51 #define VALID_DEV_OPS_OR_RET_ERR(dev, dev_id) do { \
  52         if (dev->dev_ops == NULL) { \
  53                 rte_bbdev_log(ERR, "NULL dev_ops structure in device %u", \
  54                                 dev_id); \
  55                 return -ENODEV; \
  56         } \
  57 } while (0)
  58
  59 /* Helper macro to check that driver implements required function pointer */
  60 #define VALID_FUNC_OR_RET_ERR(func, dev_id) do { \
  61         if (func == NULL) { \
  62                 rte_bbdev_log(ERR, "device %u does not support %s", \
  63                                 dev_id, #func); \
  64                 return -ENOTSUP; \
  65         } \
  66 } while (0)
  67
  68 /* Helper macro to check that queue is valid */
  69 #define VALID_QUEUE_OR_RET_ERR(queue_id, dev) do { \
  70         if (queue_id >= dev->data->num_queues) { \
  71                 rte_bbdev_log(ERR, "Invalid queue_id %u for device %u", \
  72                                 queue_id, dev->data->dev_id); \
  73                 return -ERANGE; \
  74         } \
  75 } while (0)
  76
  77 /* List of callback functions registered by an application */
  78 struct rte_bbdev_callback {
  79         TAILQ_ENTRY(rte_bbdev_callback) next;  /* Callbacks list */
  80         rte_bbdev_cb_fn cb_fn;  /* Callback address */
  81         void *cb_arg;  /* Parameter for callback */
  82         void *ret_param;  /* Return parameter */
  83         enum rte_bbdev_event_type event; /* Interrupt event type */
  84         uint32_t active; /* Callback is executing */
  85 };
  86
  87 /* spinlock for bbdev device callbacks */
  88 static rte_spinlock_t rte_bbdev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  89
  90 /*
  91  * Global array of all devices. This is not static because it's used by the
  92  * inline enqueue and dequeue functions
  93  */
  94 struct rte_bbdev rte_bbdev_devices[RTE_BBDEV_MAX_DEVS];
  95
  96 /* Global array with rte_bbdev_data structures */
  97 static struct rte_bbdev_data *rte_bbdev_data;
  98
  99 /* Memzone name for global bbdev data pool */
 100 static const char *MZ_RTE_BBDEV_DATA = "rte_bbdev_data";
 101
 102 /* Number of currently valid devices */
 103 static uint16_t num_devs;
 104
 105 /* Return pointer to device structure, with validity check */
 106 static struct rte_bbdev *
 107 get_dev(uint16_t dev_id)
 108 {
 109         if (rte_bbdev_is_valid(dev_id))
 110                 return &rte_bbdev_devices[dev_id];
 111         return NULL;
 112 }
 113
 114 /* Allocate global data array */
 115 static int
 116 rte_bbdev_data_alloc(void)
 117 {
 118         const unsigned int flags = 0;
 119         const struct rte_memzone *mz;
 120
 121         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 122                 mz = rte_memzone_reserve(MZ_RTE_BBDEV_DATA,
 123                                 RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data),
 124                                 rte_socket_id(), flags);
 125         } else
 126                 mz = rte_memzone_lookup(MZ_RTE_BBDEV_DATA);
 127         if (mz == NULL) {
 128                 rte_bbdev_log(CRIT,
 129                                 "Cannot allocate memzone for bbdev port data");
 130                 return -ENOMEM;
 131         }
 132
 133         rte_bbdev_data = mz->addr;
 134         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 135                 memset(rte_bbdev_data, 0,
 136                                 RTE_BBDEV_MAX_DEVS * sizeof(*rte_bbdev_data));
 137         return 0;
 138 }
 139
 140 /*
 141  * Find data alocated for the device or if not found return first unused bbdev
 142  * data. If all structures are in use and none is used by the device return
 143  * NULL.
 144  */
 145 static struct rte_bbdev_data *
 146 find_bbdev_data(const char *name)
 147 {
 148         uint16_t data_id;
 149
 150         for (data_id = 0; data_id < RTE_BBDEV_MAX_DEVS; ++data_id) {
 151                 if (strlen(rte_bbdev_data[data_id].name) == 0) {
 152                         memset(&rte_bbdev_data[data_id], 0,
 153                                         sizeof(struct rte_bbdev_data));
 154                         return &rte_bbdev_data[data_id];
 155                 } else if (strncmp(rte_bbdev_data[data_id].name, name,
 156                                 RTE_BBDEV_NAME_MAX_LEN) == 0)
 157                         return &rte_bbdev_data[data_id];
 158         }
 159
 160         return NULL;
 161 }
 162
 163 /* Find lowest device id with no attached device */
 164 static uint16_t
 165 find_free_dev_id(void)
 166 {
 167         uint16_t i;
 168         for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
 169                 if (rte_bbdev_devices[i].state == RTE_BBDEV_UNUSED)
 170                         return i;
 171         }
 172         return RTE_BBDEV_MAX_DEVS;
 173 }
 174
 175 struct rte_bbdev * __rte_experimental
 176 rte_bbdev_allocate(const char *name)
 177 {
 178         int ret;
 179         struct rte_bbdev *bbdev;
 180         uint16_t dev_id;
 181
 182         if (name == NULL) {
 183                 rte_bbdev_log(ERR, "Invalid null device name");
 184                 return NULL;
 185         }
 186
 187         if (rte_bbdev_get_named_dev(name) != NULL) {
 188                 rte_bbdev_log(ERR, "Device \"%s\" is already allocated", name);
 189                 return NULL;
 190         }
 191
 192         dev_id = find_free_dev_id();
 193         if (dev_id == RTE_BBDEV_MAX_DEVS) {
 194                 rte_bbdev_log(ERR, "Reached maximum number of devices");
 195                 return NULL;
 196         }
 197
 198         bbdev = &rte_bbdev_devices[dev_id];
 199
 200         if (rte_bbdev_data == NULL) {
 201                 ret = rte_bbdev_data_alloc();
 202                 if (ret != 0)
 203                         return NULL;
 204         }
 205
 206         bbdev->data = find_bbdev_data(name);
 207         if (bbdev->data == NULL) {
 208                 rte_bbdev_log(ERR,
 209                                 "Max BBDevs already allocated in multi-process environment!");
 210                 return NULL;
 211         }
 212
 213         rte_atomic16_inc(&bbdev->data->process_cnt);
 214         bbdev->data->dev_id = dev_id;
 215         bbdev->state = RTE_BBDEV_INITIALIZED;
 216
 217         ret = snprintf(bbdev->data->name, RTE_BBDEV_NAME_MAX_LEN, "%s", name);
 218         if ((ret < 0) || (ret >= RTE_BBDEV_NAME_MAX_LEN)) {
 219                 rte_bbdev_log(ERR, "Copying device name \"%s\" failed", name);
 220                 return NULL;
 221         }
 222
 223         /* init user callbacks */
 224         TAILQ_INIT(&(bbdev->list_cbs));
 225
 226         num_devs++;
 227
 228         rte_bbdev_log_debug("Initialised device %s (id = %u). Num devices = %u",
 229                         name, dev_id, num_devs);
 230
 231         return bbdev;
 232 }
 233
 234 int __rte_experimental
 235 rte_bbdev_release(struct rte_bbdev *bbdev)
 236 {
 237         uint16_t dev_id;
 238         struct rte_bbdev_callback *cb, *next;
 239
 240         if (bbdev == NULL) {
 241                 rte_bbdev_log(ERR, "NULL bbdev");
 242                 return -ENODEV;
 243         }
 244         dev_id = bbdev->data->dev_id;
 245
 246         /* free all callbacks from the device's list */
 247         for (cb = TAILQ_FIRST(&bbdev->list_cbs); cb != NULL; cb = next) {
 248
 249                 next = TAILQ_NEXT(cb, next);
 250                 TAILQ_REMOVE(&(bbdev->list_cbs), cb, next);
 251                 rte_free(cb);
 252         }
 253
 254         /* clear shared BBDev Data if no process is using the device anymore */
 255         if (rte_atomic16_dec_and_test(&bbdev->data->process_cnt))
 256                 memset(bbdev->data, 0, sizeof(*bbdev->data));
 257
 258         memset(bbdev, 0, sizeof(*bbdev));
 259         num_devs--;
 260         bbdev->state = RTE_BBDEV_UNUSED;
 261
 262         rte_bbdev_log_debug(
 263                         "Un-initialised device id = %u. Num devices = %u",
 264                         dev_id, num_devs);
 265         return 0;
 266 }
 267
 268 struct rte_bbdev * __rte_experimental
 269 rte_bbdev_get_named_dev(const char *name)
 270 {
 271         unsigned int i;
 272
 273         if (name == NULL) {
 274                 rte_bbdev_log(ERR, "NULL driver name");
 275                 return NULL;
 276         }
 277
 278         for (i = 0; i < RTE_BBDEV_MAX_DEVS; i++) {
 279                 struct rte_bbdev *dev = get_dev(i);
 280                 if (dev && (strncmp(dev->data->name,
 281                                 name, RTE_BBDEV_NAME_MAX_LEN) == 0))
 282                         return dev;
 283         }
 284
 285         return NULL;
 286 }
 287
 288 uint16_t __rte_experimental
 289 rte_bbdev_count(void)
 290 {
 291         return num_devs;
 292 }
 293
 294 bool __rte_experimental
 295 rte_bbdev_is_valid(uint16_t dev_id)
 296 {
 297         if ((dev_id < RTE_BBDEV_MAX_DEVS) &&
 298                 rte_bbdev_devices[dev_id].state == RTE_BBDEV_INITIALIZED)
 299                 return true;
 300         return false;
 301 }
 302
 303 uint16_t __rte_experimental
 304 rte_bbdev_find_next(uint16_t dev_id)
 305 {
 306         dev_id++;
 307         for (; dev_id < RTE_BBDEV_MAX_DEVS; dev_id++)
 308                 if (rte_bbdev_is_valid(dev_id))
 309                         break;
 310         return dev_id;
 311 }
 312
 313 int __rte_experimental
 314 rte_bbdev_setup_queues(uint16_t dev_id, uint16_t num_queues, int socket_id)
 315 {
 316         unsigned int i;
 317         int ret;
 318         struct rte_bbdev_driver_info dev_info;
 319         struct rte_bbdev *dev = get_dev(dev_id);
 320         VALID_DEV_OR_RET_ERR(dev, dev_id);
 321
 322         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 323
 324         if (dev->data->started) {
 325                 rte_bbdev_log(ERR,
 326                                 "Device %u cannot be configured when started",
 327                                 dev_id);
 328                 return -EBUSY;
 329         }
 330
 331         /* Get device driver information to get max number of queues */
 332         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
 333         memset(&dev_info, 0, sizeof(dev_info));
 334         dev->dev_ops->info_get(dev, &dev_info);
 335
 336         if ((num_queues == 0) || (num_queues > dev_info.max_num_queues)) {
 337                 rte_bbdev_log(ERR,
 338                                 "Device %u supports 0 < N <= %u queues, not %u",
 339                                 dev_id, dev_info.max_num_queues, num_queues);
 340                 return -EINVAL;
 341         }
 342
 343         /* If re-configuration, get driver to free existing internal memory */
 344         if (dev->data->queues != NULL) {
 345                 VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
 346                 for (i = 0; i < dev->data->num_queues; i++) {
 347                         int ret = dev->dev_ops->queue_release(dev, i);
 348                         if (ret < 0) {
 349                                 rte_bbdev_log(ERR,
 350                                                 "Device %u queue %u release failed",
 351                                                 dev_id, i);
 352                                 return ret;
 353                         }
 354                 }
 355                 /* Call optional device close */
 356                 if (dev->dev_ops->close) {
 357                         ret = dev->dev_ops->close(dev);
 358                         if (ret < 0) {
 359                                 rte_bbdev_log(ERR,
 360                                                 "Device %u couldn't be closed",
 361                                                 dev_id);
 362                                 return ret;
 363                         }
 364                 }
 365                 rte_free(dev->data->queues);
 366         }
 367
 368         /* Allocate queue pointers */
 369         dev->data->queues = rte_calloc_socket(DEV_NAME, num_queues,
 370                         sizeof(dev->data->queues[0]), RTE_CACHE_LINE_SIZE,
 371                                 dev->data->socket_id);
 372         if (dev->data->queues == NULL) {
 373                 rte_bbdev_log(ERR,
 374                                 "calloc of %u queues for device %u on socket %i failed",
 375                                 num_queues, dev_id, dev->data->socket_id);
 376                 return -ENOMEM;
 377         }
 378
 379         dev->data->num_queues = num_queues;
 380
 381         /* Call optional device configuration */
 382         if (dev->dev_ops->setup_queues) {
 383                 ret = dev->dev_ops->setup_queues(dev, num_queues, socket_id);
 384                 if (ret < 0) {
 385                         rte_bbdev_log(ERR,
 386                                         "Device %u memory configuration failed",
 387                                         dev_id);
 388                         goto error;
 389                 }
 390         }
 391
 392         rte_bbdev_log_debug("Device %u set up with %u queues", dev_id,
 393                         num_queues);
 394         return 0;
 395
 396 error:
 397         dev->data->num_queues = 0;
 398         rte_free(dev->data->queues);
 399         dev->data->queues = NULL;
 400         return ret;
 401 }
 402
 403 int __rte_experimental
 404 rte_bbdev_intr_enable(uint16_t dev_id)
 405 {
 406         int ret;
 407         struct rte_bbdev *dev = get_dev(dev_id);
 408         VALID_DEV_OR_RET_ERR(dev, dev_id);
 409
 410         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 411
 412         if (dev->data->started) {
 413                 rte_bbdev_log(ERR,
 414                                 "Device %u cannot be configured when started",
 415                                 dev_id);
 416                 return -EBUSY;
 417         }
 418
 419         if (dev->dev_ops->intr_enable) {
 420                 ret = dev->dev_ops->intr_enable(dev);
 421                 if (ret < 0) {
 422                         rte_bbdev_log(ERR,
 423                                         "Device %u interrupts configuration failed",
 424                                         dev_id);
 425                         return ret;
 426                 }
 427                 rte_bbdev_log_debug("Enabled interrupts for dev %u", dev_id);
 428                 return 0;
 429         }
 430
 431         rte_bbdev_log(ERR, "Device %u doesn't support interrupts", dev_id);
 432         return -ENOTSUP;
 433 }
 434
 435 int __rte_experimental
 436 rte_bbdev_queue_configure(uint16_t dev_id, uint16_t queue_id,
 437                 const struct rte_bbdev_queue_conf *conf)
 438 {
 439         int ret = 0;
 440         struct rte_bbdev_driver_info dev_info;
 441         struct rte_bbdev *dev = get_dev(dev_id);
 442         const struct rte_bbdev_op_cap *p;
 443         struct rte_bbdev_queue_conf *stored_conf;
 444         const char *op_type_str;
 445         VALID_DEV_OR_RET_ERR(dev, dev_id);
 446
 447         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 448
 449         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 450
 451         if (dev->data->queues[queue_id].started || dev->data->started) {
 452                 rte_bbdev_log(ERR,
 453                                 "Queue %u of device %u cannot be configured when started",
 454                                 queue_id, dev_id);
 455                 return -EBUSY;
 456         }
 457
 458         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_release, dev_id);
 459         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_setup, dev_id);
 460
 461         /* Get device driver information to verify config is valid */
 462         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
 463         memset(&dev_info, 0, sizeof(dev_info));
 464         dev->dev_ops->info_get(dev, &dev_info);
 465
 466         /* Check configuration is valid */
 467         if (conf != NULL) {
 468                 if ((conf->op_type == RTE_BBDEV_OP_NONE) &&
 469                                 (dev_info.capabilities[0].type ==
 470                                 RTE_BBDEV_OP_NONE)) {
 471                         ret = 1;
 472                 } else {
 473                         for (p = dev_info.capabilities;
 474                                         p->type != RTE_BBDEV_OP_NONE; p++) {
 475                                 if (conf->op_type == p->type) {
 476                                         ret = 1;
 477                                         break;
 478                                 }
 479                         }
 480                 }
 481                 if (ret == 0) {
 482                         rte_bbdev_log(ERR, "Invalid operation type");
 483                         return -EINVAL;
 484                 }
 485                 if (conf->queue_size > dev_info.queue_size_lim) {
 486                         rte_bbdev_log(ERR,
 487                                         "Size (%u) of queue %u of device %u must be: <= %u",
 488                                         conf->queue_size, queue_id, dev_id,
 489                                         dev_info.queue_size_lim);
 490                         return -EINVAL;
 491                 }
 492                 if (!rte_is_power_of_2(conf->queue_size)) {
 493                         rte_bbdev_log(ERR,
 494                                         "Size (%u) of queue %u of device %u must be a power of 2",
 495                                         conf->queue_size, queue_id, dev_id);
 496                         return -EINVAL;
 497                 }
 498                 if (conf->priority > dev_info.max_queue_priority) {
 499                         rte_bbdev_log(ERR,
 500                                         "Priority (%u) of queue %u of bdev %u must be <= %u",
 501                                         conf->priority, queue_id, dev_id,
 502                                         dev_info.max_queue_priority);
 503                         return -EINVAL;
 504                 }
 505         }
 506
 507         /* Release existing queue (in case of queue reconfiguration) */
 508         if (dev->data->queues[queue_id].queue_private != NULL) {
 509                 ret = dev->dev_ops->queue_release(dev, queue_id);
 510                 if (ret < 0) {
 511                         rte_bbdev_log(ERR, "Device %u queue %u release failed",
 512                                         dev_id, queue_id);
 513                         return ret;
 514                 }
 515         }
 516
 517         /* Get driver to setup the queue */
 518         ret = dev->dev_ops->queue_setup(dev, queue_id, (conf != NULL) ?
 519                         conf : &dev_info.default_queue_conf);
 520         if (ret < 0) {
 521                 rte_bbdev_log(ERR,
 522                                 "Device %u queue %u setup failed", dev_id,
 523                                 queue_id);
 524                 return ret;
 525         }
 526
 527         /* Store configuration */
 528         stored_conf = &dev->data->queues[queue_id].conf;
 529         memcpy(stored_conf,
 530                         (conf != NULL) ? conf : &dev_info.default_queue_conf,
 531                         sizeof(*stored_conf));
 532
 533         op_type_str = rte_bbdev_op_type_str(stored_conf->op_type);
 534         if (op_type_str == NULL)
 535                 return -EINVAL;
 536
 537         rte_bbdev_log_debug("Configured dev%uq%u (size=%u, type=%s, prio=%u)",
 538                         dev_id, queue_id, stored_conf->queue_size, op_type_str,
 539                         stored_conf->priority);
 540
 541         return 0;
 542 }
 543
 544 int __rte_experimental
 545 rte_bbdev_start(uint16_t dev_id)
 546 {
 547         int i;
 548         struct rte_bbdev *dev = get_dev(dev_id);
 549         VALID_DEV_OR_RET_ERR(dev, dev_id);
 550
 551         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 552
 553         if (dev->data->started) {
 554                 rte_bbdev_log_debug("Device %u is already started", dev_id);
 555                 return 0;
 556         }
 557
 558         if (dev->dev_ops->start) {
 559                 int ret = dev->dev_ops->start(dev);
 560                 if (ret < 0) {
 561                         rte_bbdev_log(ERR, "Device %u start failed", dev_id);
 562                         return ret;
 563                 }
 564         }
 565
 566         /* Store new state */
 567         for (i = 0; i < dev->data->num_queues; i++)
 568                 if (!dev->data->queues[i].conf.deferred_start)
 569                         dev->data->queues[i].started = true;
 570         dev->data->started = true;
 571
 572         rte_bbdev_log_debug("Started device %u", dev_id);
 573         return 0;
 574 }
 575
 576 int __rte_experimental
 577 rte_bbdev_stop(uint16_t dev_id)
 578 {
 579         struct rte_bbdev *dev = get_dev(dev_id);
 580         VALID_DEV_OR_RET_ERR(dev, dev_id);
 581
 582         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 583
 584         if (!dev->data->started) {
 585                 rte_bbdev_log_debug("Device %u is already stopped", dev_id);
 586                 return 0;
 587         }
 588
 589         if (dev->dev_ops->stop)
 590                 dev->dev_ops->stop(dev);
 591         dev->data->started = false;
 592
 593         rte_bbdev_log_debug("Stopped device %u", dev_id);
 594         return 0;
 595 }
 596
 597 int __rte_experimental
 598 rte_bbdev_close(uint16_t dev_id)
 599 {
 600         int ret;
 601         uint16_t i;
 602         struct rte_bbdev *dev = get_dev(dev_id);
 603         VALID_DEV_OR_RET_ERR(dev, dev_id);
 604
 605         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 606
 607         if (dev->data->started) {
 608                 ret = rte_bbdev_stop(dev_id);
 609                 if (ret < 0) {
 610                         rte_bbdev_log(ERR, "Device %u stop failed", dev_id);
 611                         return ret;
 612                 }
 613         }
 614
 615         /* Free memory used by queues */
 616         for (i = 0; i < dev->data->num_queues; i++) {
 617                 ret = dev->dev_ops->queue_release(dev, i);
 618                 if (ret < 0) {
 619                         rte_bbdev_log(ERR, "Device %u queue %u release failed",
 620                                         dev_id, i);
 621                         return ret;
 622                 }
 623         }
 624         rte_free(dev->data->queues);
 625
 626         if (dev->dev_ops->close) {
 627                 ret = dev->dev_ops->close(dev);
 628                 if (ret < 0) {
 629                         rte_bbdev_log(ERR, "Device %u close failed", dev_id);
 630                         return ret;
 631                 }
 632         }
 633
 634         /* Clear configuration */
 635         dev->data->queues = NULL;
 636         dev->data->num_queues = 0;
 637
 638         rte_bbdev_log_debug("Closed device %u", dev_id);
 639         return 0;
 640 }
 641
 642 int __rte_experimental
 643 rte_bbdev_queue_start(uint16_t dev_id, uint16_t queue_id)
 644 {
 645         struct rte_bbdev *dev = get_dev(dev_id);
 646         VALID_DEV_OR_RET_ERR(dev, dev_id);
 647
 648         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 649
 650         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 651
 652         if (dev->data->queues[queue_id].started) {
 653                 rte_bbdev_log_debug("Queue %u of device %u already started",
 654                                 queue_id, dev_id);
 655                 return 0;
 656         }
 657
 658         if (dev->dev_ops->queue_start) {
 659                 int ret = dev->dev_ops->queue_start(dev, queue_id);
 660                 if (ret < 0) {
 661                         rte_bbdev_log(ERR, "Device %u queue %u start failed",
 662                                         dev_id, queue_id);
 663                         return ret;
 664                 }
 665         }
 666         dev->data->queues[queue_id].started = true;
 667
 668         rte_bbdev_log_debug("Started queue %u of device %u", queue_id, dev_id);
 669         return 0;
 670 }
 671
 672 int __rte_experimental
 673 rte_bbdev_queue_stop(uint16_t dev_id, uint16_t queue_id)
 674 {
 675         struct rte_bbdev *dev = get_dev(dev_id);
 676         VALID_DEV_OR_RET_ERR(dev, dev_id);
 677
 678         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 679
 680         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 681
 682         if (!dev->data->queues[queue_id].started) {
 683                 rte_bbdev_log_debug("Queue %u of device %u already stopped",
 684                                 queue_id, dev_id);
 685                 return 0;
 686         }
 687
 688         if (dev->dev_ops->queue_stop) {
 689                 int ret = dev->dev_ops->queue_stop(dev, queue_id);
 690                 if (ret < 0) {
 691                         rte_bbdev_log(ERR, "Device %u queue %u stop failed",
 692                                         dev_id, queue_id);
 693                         return ret;
 694                 }
 695         }
 696         dev->data->queues[queue_id].started = false;
 697
 698         rte_bbdev_log_debug("Stopped queue %u of device %u", queue_id, dev_id);
 699         return 0;
 700 }
 701
 702 /* Get device statistics */
 703 static void
 704 get_stats_from_queues(struct rte_bbdev *dev, struct rte_bbdev_stats *stats)
 705 {
 706         unsigned int q_id;
 707         for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
 708                 struct rte_bbdev_stats *q_stats =
 709                                 &dev->data->queues[q_id].queue_stats;
 710
 711                 stats->enqueued_count += q_stats->enqueued_count;
 712                 stats->dequeued_count += q_stats->dequeued_count;
 713                 stats->enqueue_err_count += q_stats->enqueue_err_count;
 714                 stats->dequeue_err_count += q_stats->dequeue_err_count;
 715         }
 716         rte_bbdev_log_debug("Got stats on %u", dev->data->dev_id);
 717 }
 718
 719 static void
 720 reset_stats_in_queues(struct rte_bbdev *dev)
 721 {
 722         unsigned int q_id;
 723         for (q_id = 0; q_id < dev->data->num_queues; q_id++) {
 724                 struct rte_bbdev_stats *q_stats =
 725                                 &dev->data->queues[q_id].queue_stats;
 726
 727                 memset(q_stats, 0, sizeof(*q_stats));
 728         }
 729         rte_bbdev_log_debug("Reset stats on %u", dev->data->dev_id);
 730 }
 731
 732 int __rte_experimental
 733 rte_bbdev_stats_get(uint16_t dev_id, struct rte_bbdev_stats *stats)
 734 {
 735         struct rte_bbdev *dev = get_dev(dev_id);
 736         VALID_DEV_OR_RET_ERR(dev, dev_id);
 737
 738         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 739
 740         if (stats == NULL) {
 741                 rte_bbdev_log(ERR, "NULL stats structure");
 742                 return -EINVAL;
 743         }
 744
 745         memset(stats, 0, sizeof(*stats));
 746         if (dev->dev_ops->stats_get != NULL)
 747                 dev->dev_ops->stats_get(dev, stats);
 748         else
 749                 get_stats_from_queues(dev, stats);
 750
 751         rte_bbdev_log_debug("Retrieved stats of device %u", dev_id);
 752         return 0;
 753 }
 754
 755 int __rte_experimental
 756 rte_bbdev_stats_reset(uint16_t dev_id)
 757 {
 758         struct rte_bbdev *dev = get_dev(dev_id);
 759         VALID_DEV_OR_RET_ERR(dev, dev_id);
 760
 761         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
 762
 763         if (dev->dev_ops->stats_reset != NULL)
 764                 dev->dev_ops->stats_reset(dev);
 765         else
 766                 reset_stats_in_queues(dev);
 767
 768         rte_bbdev_log_debug("Reset stats of device %u", dev_id);
 769         return 0;
 770 }
 771
 772 int __rte_experimental
 773 rte_bbdev_info_get(uint16_t dev_id, struct rte_bbdev_info *dev_info)
 774 {
 775         struct rte_bbdev *dev = get_dev(dev_id);
 776         VALID_DEV_OR_RET_ERR(dev, dev_id);
 777
 778         VALID_FUNC_OR_RET_ERR(dev->dev_ops->info_get, dev_id);
 779
 780         if (dev_info == NULL) {
 781                 rte_bbdev_log(ERR, "NULL dev info structure");
 782                 return -EINVAL;
 783         }
 784
 785         /* Copy data maintained by device interface layer */
 786         memset(dev_info, 0, sizeof(*dev_info));
 787         dev_info->dev_name = dev->data->name;
 788         dev_info->num_queues = dev->data->num_queues;
 789         dev_info->bus = rte_bus_find_by_device(dev->device);
 790         dev_info->socket_id = dev->data->socket_id;
 791         dev_info->started = dev->data->started;
 792
 793         /* Copy data maintained by device driver layer */
 794         dev->dev_ops->info_get(dev, &dev_info->drv);
 795
 796         rte_bbdev_log_debug("Retrieved info of device %u", dev_id);
 797         return 0;
 798 }
 799
 800 int __rte_experimental
 801 rte_bbdev_queue_info_get(uint16_t dev_id, uint16_t queue_id,
 802                 struct rte_bbdev_queue_info *queue_info)
 803 {
 804         struct rte_bbdev *dev = get_dev(dev_id);
 805         VALID_DEV_OR_RET_ERR(dev, dev_id);
 806
 807         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
 808
 809         if (queue_info == NULL) {
 810                 rte_bbdev_log(ERR, "NULL queue info structure");
 811                 return -EINVAL;
 812         }
 813
 814         /* Copy data to output */
 815         memset(queue_info, 0, sizeof(*queue_info));
 816         queue_info->conf = dev->data->queues[queue_id].conf;
 817         queue_info->started = dev->data->queues[queue_id].started;
 818
 819         rte_bbdev_log_debug("Retrieved info of queue %u of device %u",
 820                         queue_id, dev_id);
 821         return 0;
 822 }
 823
 824 /* Calculate size needed to store bbdev_op, depending on type */
 825 static unsigned int
 826 get_bbdev_op_size(enum rte_bbdev_op_type type)
 827 {
 828         unsigned int result = 0;
 829         switch (type) {
 830         case RTE_BBDEV_OP_NONE:
 831                 result = RTE_MAX(sizeof(struct rte_bbdev_dec_op),
 832                                 sizeof(struct rte_bbdev_enc_op));
 833                 break;
 834         case RTE_BBDEV_OP_TURBO_DEC:
 835                 result = sizeof(struct rte_bbdev_dec_op);
 836                 break;
 837         case RTE_BBDEV_OP_TURBO_ENC:
 838                 result = sizeof(struct rte_bbdev_enc_op);
 839                 break;
 840         default:
 841                 break;
 842         }
 843
 844         return result;
 845 }
 846
 847 /* Initialise a bbdev_op structure */
 848 static void
 849 bbdev_op_init(struct rte_mempool *mempool, void *arg, void *element,
 850                 __rte_unused unsigned int n)
 851 {
 852         enum rte_bbdev_op_type type = *(enum rte_bbdev_op_type *)arg;
 853
 854         if (type == RTE_BBDEV_OP_TURBO_DEC) {
 855                 struct rte_bbdev_dec_op *op = element;
 856                 memset(op, 0, mempool->elt_size);
 857                 op->mempool = mempool;
 858         } else if (type == RTE_BBDEV_OP_TURBO_ENC) {
 859                 struct rte_bbdev_enc_op *op = element;
 860                 memset(op, 0, mempool->elt_size);
 861                 op->mempool = mempool;
 862         }
 863 }
 864
 865 struct rte_mempool * __rte_experimental
 866 rte_bbdev_op_pool_create(const char *name, enum rte_bbdev_op_type type,
 867                 unsigned int num_elements, unsigned int cache_size,
 868                 int socket_id)
 869 {
 870         struct rte_bbdev_op_pool_private *priv;
 871         struct rte_mempool *mp;
 872         const char *op_type_str;
 873
 874         if (name == NULL) {
 875                 rte_bbdev_log(ERR, "NULL name for op pool");
 876                 return NULL;
 877         }
 878
 879         if (type >= RTE_BBDEV_OP_TYPE_COUNT) {
 880                 rte_bbdev_log(ERR,
 881                                 "Invalid op type (%u), should be less than %u",
 882                                 type, RTE_BBDEV_OP_TYPE_COUNT);
 883                 return NULL;
 884         }
 885
 886         mp = rte_mempool_create(name, num_elements, get_bbdev_op_size(type),
 887                         cache_size, sizeof(struct rte_bbdev_op_pool_private),
 888                         NULL, NULL, bbdev_op_init, &type, socket_id, 0);
 889         if (mp == NULL) {
 890                 rte_bbdev_log(ERR,
 891                                 "Failed to create op pool %s (num ops=%u, op size=%u) with error: %s",
 892                                 name, num_elements, get_bbdev_op_size(type),
 893                                 rte_strerror(rte_errno));
 894                 return NULL;
 895         }
 896
 897         op_type_str = rte_bbdev_op_type_str(type);
 898         if (op_type_str == NULL)
 899                 return NULL;
 900
 901         rte_bbdev_log_debug(
 902                         "Op pool %s created for %u ops (type=%s, cache=%u, socket=%u, size=%u)",
 903                         name, num_elements, op_type_str, cache_size, socket_id,
 904                         get_bbdev_op_size(type));
 905
 906         priv = (struct rte_bbdev_op_pool_private *)rte_mempool_get_priv(mp);
 907         priv->type = type;
 908
 909         return mp;
 910 }
 911
 912 int __rte_experimental
 913 rte_bbdev_callback_register(uint16_t dev_id, enum rte_bbdev_event_type event,
 914                 rte_bbdev_cb_fn cb_fn, void *cb_arg)
 915 {
 916         struct rte_bbdev_callback *user_cb;
 917         struct rte_bbdev *dev = get_dev(dev_id);
 918         VALID_DEV_OR_RET_ERR(dev, dev_id);
 919
 920         if (event >= RTE_BBDEV_EVENT_MAX) {
 921                 rte_bbdev_log(ERR,
 922                                 "Invalid event type (%u), should be less than %u",
 923                                 event, RTE_BBDEV_EVENT_MAX);
 924                 return -EINVAL;
 925         }
 926
 927         if (cb_fn == NULL) {
 928                 rte_bbdev_log(ERR, "NULL callback function");
 929                 return -EINVAL;
 930         }
 931
 932         rte_spinlock_lock(&rte_bbdev_cb_lock);
 933
 934         TAILQ_FOREACH(user_cb, &(dev->list_cbs), next) {
 935                 if (user_cb->cb_fn == cb_fn &&
 936                                 user_cb->cb_arg == cb_arg &&
 937                                 user_cb->event == event)
 938                         break;
 939         }
 940
 941         /* create a new callback. */
 942         if (user_cb == NULL) {
 943                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
 944                                 sizeof(struct rte_bbdev_callback), 0);
 945                 if (user_cb != NULL) {
 946                         user_cb->cb_fn = cb_fn;
 947                         user_cb->cb_arg = cb_arg;
 948                         user_cb->event = event;
 949                         TAILQ_INSERT_TAIL(&(dev->list_cbs), user_cb, next);
 950                 }
 951         }
 952
 953         rte_spinlock_unlock(&rte_bbdev_cb_lock);
 954         return (user_cb == NULL) ? -ENOMEM : 0;
 955 }
 956
 957 int __rte_experimental
 958 rte_bbdev_callback_unregister(uint16_t dev_id, enum rte_bbdev_event_type event,
 959                 rte_bbdev_cb_fn cb_fn, void *cb_arg)
 960 {
 961         int ret = 0;
 962         struct rte_bbdev_callback *cb, *next;
 963         struct rte_bbdev *dev = get_dev(dev_id);
 964         VALID_DEV_OR_RET_ERR(dev, dev_id);
 965
 966         if (event >= RTE_BBDEV_EVENT_MAX) {
 967                 rte_bbdev_log(ERR,
 968                                 "Invalid event type (%u), should be less than %u",
 969                                 event, RTE_BBDEV_EVENT_MAX);
 970                 return -EINVAL;
 971         }
 972
 973         if (cb_fn == NULL) {
 974                 rte_bbdev_log(ERR,
 975                                 "NULL callback function cannot be unregistered");
 976                 return -EINVAL;
 977         }
 978
 979         dev = &rte_bbdev_devices[dev_id];
 980         rte_spinlock_lock(&rte_bbdev_cb_lock);
 981
 982         for (cb = TAILQ_FIRST(&dev->list_cbs); cb != NULL; cb = next) {
 983
 984                 next = TAILQ_NEXT(cb, next);
 985
 986                 if (cb->cb_fn != cb_fn || cb->event != event ||
 987                                 (cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
 988                         continue;
 989
 990                 /* If this callback is not executing right now, remove it. */
 991                 if (cb->active == 0) {
 992                         TAILQ_REMOVE(&(dev->list_cbs), cb, next);
 993                         rte_free(cb);
 994                 } else
 995                         ret = -EAGAIN;
 996         }
 997
 998         rte_spinlock_unlock(&rte_bbdev_cb_lock);
 999         return ret;
1000 }
1001
1002 void __rte_experimental
1003 rte_bbdev_pmd_callback_process(struct rte_bbdev *dev,
1004         enum rte_bbdev_event_type event, void *ret_param)
1005 {
1006         struct rte_bbdev_callback *cb_lst;
1007         struct rte_bbdev_callback dev_cb;
1008
1009         if (dev == NULL) {
1010                 rte_bbdev_log(ERR, "NULL device");
1011                 return;
1012         }
1013
1014         if (dev->data == NULL) {
1015                 rte_bbdev_log(ERR, "NULL data structure");
1016                 return;
1017         }
1018
1019         if (event >= RTE_BBDEV_EVENT_MAX) {
1020                 rte_bbdev_log(ERR,
1021                                 "Invalid event type (%u), should be less than %u",
1022                                 event, RTE_BBDEV_EVENT_MAX);
1023                 return;
1024         }
1025
1026         rte_spinlock_lock(&rte_bbdev_cb_lock);
1027         TAILQ_FOREACH(cb_lst, &(dev->list_cbs), next) {
1028                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
1029                         continue;
1030                 dev_cb = *cb_lst;
1031                 cb_lst->active = 1;
1032                 if (ret_param != NULL)
1033                         dev_cb.ret_param = ret_param;
1034
1035                 rte_spinlock_unlock(&rte_bbdev_cb_lock);
1036                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
1037                                 dev_cb.cb_arg, dev_cb.ret_param);
1038                 rte_spinlock_lock(&rte_bbdev_cb_lock);
1039                 cb_lst->active = 0;
1040         }
1041         rte_spinlock_unlock(&rte_bbdev_cb_lock);
1042 }
1043
1044 int __rte_experimental
1045 rte_bbdev_queue_intr_enable(uint16_t dev_id, uint16_t queue_id)
1046 {
1047         struct rte_bbdev *dev = get_dev(dev_id);
1048         VALID_DEV_OR_RET_ERR(dev, dev_id);
1049         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1050         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
1051         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_enable, dev_id);
1052         return dev->dev_ops->queue_intr_enable(dev, queue_id);
1053 }
1054
1055 int __rte_experimental
1056 rte_bbdev_queue_intr_disable(uint16_t dev_id, uint16_t queue_id)
1057 {
1058         struct rte_bbdev *dev = get_dev(dev_id);
1059         VALID_DEV_OR_RET_ERR(dev, dev_id);
1060         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1061         VALID_DEV_OPS_OR_RET_ERR(dev, dev_id);
1062         VALID_FUNC_OR_RET_ERR(dev->dev_ops->queue_intr_disable, dev_id);
1063         return dev->dev_ops->queue_intr_disable(dev, queue_id);
1064 }
1065
1066 int __rte_experimental
1067 rte_bbdev_queue_intr_ctl(uint16_t dev_id, uint16_t queue_id, int epfd, int op,
1068                 void *data)
1069 {
1070         uint32_t vec;
1071         struct rte_bbdev *dev = get_dev(dev_id);
1072         struct rte_intr_handle *intr_handle;
1073         int ret;
1074
1075         VALID_DEV_OR_RET_ERR(dev, dev_id);
1076         VALID_QUEUE_OR_RET_ERR(queue_id, dev);
1077
1078         intr_handle = dev->intr_handle;
1079         if (!intr_handle || !intr_handle->intr_vec) {
1080                 rte_bbdev_log(ERR, "Device %u intr handle unset\n", dev_id);
1081                 return -ENOTSUP;
1082         }
1083
1084         if (queue_id >= RTE_MAX_RXTX_INTR_VEC_ID) {
1085                 rte_bbdev_log(ERR, "Device %u queue_id %u is too big\n",
1086                                 dev_id, queue_id);
1087                 return -ENOTSUP;
1088         }
1089
1090         vec = intr_handle->intr_vec[queue_id];
1091         ret = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
1092         if (ret && (ret != -EEXIST)) {
1093                 rte_bbdev_log(ERR,
1094                                 "dev %u q %u int ctl error op %d epfd %d vec %u\n",
1095                                 dev_id, queue_id, op, epfd, vec);
1096                 return ret;
1097         }
1098
1099         return 0;
1100 }
1101
1102
1103 const char * __rte_experimental
1104 rte_bbdev_op_type_str(enum rte_bbdev_op_type op_type)
1105 {
1106         static const char * const op_types[] = {
1107                 "RTE_BBDEV_OP_NONE",
1108                 "RTE_BBDEV_OP_TURBO_DEC",
1109                 "RTE_BBDEV_OP_TURBO_ENC",
1110         };
1111
1112         if (op_type < RTE_BBDEV_OP_TYPE_COUNT)
1113                 return op_types[op_type];
1114
1115         rte_bbdev_log(ERR, "Invalid operation type");
1116         return NULL;
1117 }
1118
1119 RTE_INIT(rte_bbdev_init_log);
1120 static void
1121 rte_bbdev_init_log(void)
1122 {
1123         bbdev_logtype = rte_log_register("lib.bbdev");
1124         if (bbdev_logtype >= 0)
1125                 rte_log_set_level(bbdev_logtype, RTE_LOG_NOTICE);
1126 }