lib/librte_cryptodev/rte_cryptodev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <sys/types.h>
  34 #include <sys/queue.h>
  35 #include <ctype.h>
  36 #include <stdio.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <stdarg.h>
  40 #include <errno.h>
  41 #include <stdint.h>
  42 #include <inttypes.h>
  43 #include <netinet/in.h>
  44
  45 #include <rte_byteorder.h>
  46 #include <rte_log.h>
  47 #include <rte_debug.h>
  48 #include <rte_dev.h>
  49 #include <rte_interrupts.h>
  50 #include <rte_pci.h>
  51 #include <rte_memory.h>
  52 #include <rte_memcpy.h>
  53 #include <rte_memzone.h>
  54 #include <rte_launch.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_per_lcore.h>
  58 #include <rte_lcore.h>
  59 #include <rte_atomic.h>
  60 #include <rte_branch_prediction.h>
  61 #include <rte_common.h>
  62 #include <rte_ring.h>
  63 #include <rte_mempool.h>
  64 #include <rte_malloc.h>
  65 #include <rte_mbuf.h>
  66 #include <rte_errno.h>
  67 #include <rte_spinlock.h>
  68 #include <rte_string_fns.h>
  69
  70 #include "rte_crypto.h"
  71 #include "rte_cryptodev.h"
  72 #include "rte_cryptodev_pmd.h"
  73
  74 struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS];
  75
  76 struct rte_cryptodev *rte_cryptodevs = &rte_crypto_devices[0];
  77
  78 static struct rte_cryptodev_global cryptodev_globals = {
  79                 .devs                   = &rte_crypto_devices[0],
  80                 .data                   = { NULL },
  81                 .nb_devs                = 0,
  82                 .max_devs               = RTE_CRYPTO_MAX_DEVS
  83 };
  84
  85 struct rte_cryptodev_global *rte_cryptodev_globals = &cryptodev_globals;
  86
  87 /* spinlock for crypto device callbacks */
  88 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  89
  90
  91 /**
  92  * The user application callback description.
  93  *
  94  * It contains callback address to be registered by user application,
  95  * the pointer to the parameters for callback, and the event type.
  96  */
  97 struct rte_cryptodev_callback {
  98         TAILQ_ENTRY(rte_cryptodev_callback) next; /**< Callbacks list */
  99         rte_cryptodev_cb_fn cb_fn;              /**< Callback address */
 100         void *cb_arg;                           /**< Parameter for callback */
 101         enum rte_cryptodev_event_type event;    /**< Interrupt event type */
 102         uint32_t active;                        /**< Callback is executing */
 103 };
 104
 105 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG                ("max_nb_queue_pairs")
 106 #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG              ("max_nb_sessions")
 107 #define RTE_CRYPTODEV_VDEV_SOCKET_ID                    ("socket_id")
 108
 109 static const char *cryptodev_vdev_valid_params[] = {
 110         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
 111         RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
 112         RTE_CRYPTODEV_VDEV_SOCKET_ID
 113 };
 114
 115 static uint8_t
 116 number_of_sockets(void)
 117 {
 118         int sockets = 0;
 119         int i;
 120         const struct rte_memseg *ms = rte_eal_get_physmem_layout();
 121
 122         for (i = 0; ((i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL)); i++) {
 123                 if (sockets < ms[i].socket_id)
 124                         sockets = ms[i].socket_id;
 125         }
 126
 127         /* Number of sockets = maximum socket_id + 1 */
 128         return ++sockets;
 129 }
 130
 131 /** Parse integer from integer argument */
 132 static int
 133 parse_integer_arg(const char *key __rte_unused,
 134                 const char *value, void *extra_args)
 135 {
 136         int *i = (int *) extra_args;
 137
 138         *i = atoi(value);
 139         if (*i < 0) {
 140                 CDEV_LOG_ERR("Argument has to be positive.");
 141                 return -1;
 142         }
 143
 144         return 0;
 145 }
 146
 147 int
 148 rte_cryptodev_parse_vdev_init_params(struct rte_crypto_vdev_init_params *params,
 149                 const char *input_args)
 150 {
 151         struct rte_kvargs *kvlist;
 152         int ret;
 153
 154         if (params == NULL)
 155                 return -EINVAL;
 156
 157         if (input_args) {
 158                 kvlist = rte_kvargs_parse(input_args,
 159                                 cryptodev_vdev_valid_params);
 160                 if (kvlist == NULL)
 161                         return -1;
 162
 163                 ret = rte_kvargs_process(kvlist,
 164                                         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
 165                                         &parse_integer_arg,
 166                                         &params->max_nb_queue_pairs);
 167                 if (ret < 0)
 168                         goto free_kvlist;
 169
 170                 ret = rte_kvargs_process(kvlist,
 171                                         RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
 172                                         &parse_integer_arg,
 173                                         &params->max_nb_sessions);
 174                 if (ret < 0)
 175                         goto free_kvlist;
 176
 177                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
 178                                         &parse_integer_arg,
 179                                         &params->socket_id);
 180                 if (ret < 0)
 181                         goto free_kvlist;
 182
 183                 if (params->socket_id >= number_of_sockets()) {
 184                         CDEV_LOG_ERR("Invalid socket id specified to create "
 185                                 "the virtual crypto device on");
 186                         goto free_kvlist;
 187                 }
 188         }
 189
 190         return 0;
 191
 192 free_kvlist:
 193         rte_kvargs_free(kvlist);
 194         return ret;
 195 }
 196
 197 const char *
 198 rte_cryptodev_get_feature_name(uint64_t flag)
 199 {
 200         switch (flag) {
 201         case RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO:
 202                 return "SYMMETRIC_CRYPTO";
 203         case RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO:
 204                 return "ASYMMETRIC_CRYPTO";
 205         case RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING:
 206                 return "SYM_OPERATION_CHAINING";
 207         case RTE_CRYPTODEV_FF_CPU_SSE:
 208                 return "CPU_SSE";
 209         case RTE_CRYPTODEV_FF_CPU_AVX:
 210                 return "CPU_AVX";
 211         case RTE_CRYPTODEV_FF_CPU_AVX2:
 212                 return "CPU_AVX2";
 213         case RTE_CRYPTODEV_FF_CPU_AESNI:
 214                 return "CPU_AESNI";
 215         case RTE_CRYPTODEV_FF_HW_ACCELERATED:
 216                 return "HW_ACCELERATED";
 217
 218         default:
 219                 return NULL;
 220         }
 221 }
 222
 223
 224 int
 225 rte_cryptodev_create_vdev(const char *name, const char *args)
 226 {
 227         return rte_eal_vdev_init(name, args);
 228 }
 229
 230 int
 231 rte_cryptodev_get_dev_id(const char *name) {
 232         unsigned i;
 233
 234         if (name == NULL)
 235                 return -1;
 236
 237         for (i = 0; i < rte_cryptodev_globals->max_devs; i++)
 238                 if ((strcmp(rte_cryptodev_globals->devs[i].data->name, name)
 239                                 == 0) &&
 240                                 (rte_cryptodev_globals->devs[i].attached ==
 241                                                 RTE_CRYPTODEV_ATTACHED))
 242                         return i;
 243
 244         return -1;
 245 }
 246
 247 uint8_t
 248 rte_cryptodev_count(void)
 249 {
 250         return rte_cryptodev_globals->nb_devs;
 251 }
 252
 253 uint8_t
 254 rte_cryptodev_count_devtype(enum rte_cryptodev_type type)
 255 {
 256         uint8_t i, dev_count = 0;
 257
 258         for (i = 0; i < rte_cryptodev_globals->max_devs; i++)
 259                 if (rte_cryptodev_globals->devs[i].dev_type == type &&
 260                         rte_cryptodev_globals->devs[i].attached ==
 261                                         RTE_CRYPTODEV_ATTACHED)
 262                         dev_count++;
 263
 264         return dev_count;
 265 }
 266
 267 int
 268 rte_cryptodev_socket_id(uint8_t dev_id)
 269 {
 270         struct rte_cryptodev *dev;
 271
 272         if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
 273                 return -1;
 274
 275         dev = rte_cryptodev_pmd_get_dev(dev_id);
 276
 277         return dev->data->socket_id;
 278 }
 279
 280 static inline int
 281 rte_cryptodev_data_alloc(uint8_t dev_id, struct rte_cryptodev_data **data,
 282                 int socket_id)
 283 {
 284         char mz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 285         const struct rte_memzone *mz;
 286         int n;
 287
 288         /* generate memzone name */
 289         n = snprintf(mz_name, sizeof(mz_name), "rte_cryptodev_data_%u", dev_id);
 290         if (n >= (int)sizeof(mz_name))
 291                 return -EINVAL;
 292
 293         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 294                 mz = rte_memzone_reserve(mz_name,
 295                                 sizeof(struct rte_cryptodev_data),
 296                                 socket_id, 0);
 297         } else
 298                 mz = rte_memzone_lookup(mz_name);
 299
 300         if (mz == NULL)
 301                 return -ENOMEM;
 302
 303         *data = mz->addr;
 304         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 305                 memset(*data, 0, sizeof(struct rte_cryptodev_data));
 306
 307         return 0;
 308 }
 309
 310 static uint8_t
 311 rte_cryptodev_find_free_device_index(void)
 312 {
 313         uint8_t dev_id;
 314
 315         for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++) {
 316                 if (rte_crypto_devices[dev_id].attached ==
 317                                 RTE_CRYPTODEV_DETACHED)
 318                         return dev_id;
 319         }
 320         return RTE_CRYPTO_MAX_DEVS;
 321 }
 322
 323 struct rte_cryptodev *
 324 rte_cryptodev_pmd_allocate(const char *name, enum pmd_type type, int socket_id)
 325 {
 326         struct rte_cryptodev *cryptodev;
 327         uint8_t dev_id;
 328
 329         if (rte_cryptodev_pmd_get_named_dev(name) != NULL) {
 330                 CDEV_LOG_ERR("Crypto device with name %s already "
 331                                 "allocated!", name);
 332                 return NULL;
 333         }
 334
 335         dev_id = rte_cryptodev_find_free_device_index();
 336         if (dev_id == RTE_CRYPTO_MAX_DEVS) {
 337                 CDEV_LOG_ERR("Reached maximum number of crypto devices");
 338                 return NULL;
 339         }
 340
 341         cryptodev = rte_cryptodev_pmd_get_dev(dev_id);
 342
 343         if (cryptodev->data == NULL) {
 344                 struct rte_cryptodev_data *cryptodev_data =
 345                                 cryptodev_globals.data[dev_id];
 346
 347                 int retval = rte_cryptodev_data_alloc(dev_id, &cryptodev_data,
 348                                 socket_id);
 349
 350                 if (retval < 0 || cryptodev_data == NULL)
 351                         return NULL;
 352
 353                 cryptodev->data = cryptodev_data;
 354
 355                 snprintf(cryptodev->data->name, RTE_CRYPTODEV_NAME_MAX_LEN,
 356                                 "%s", name);
 357
 358                 cryptodev->data->dev_id = dev_id;
 359                 cryptodev->data->socket_id = socket_id;
 360                 cryptodev->data->dev_started = 0;
 361
 362                 cryptodev->attached = RTE_CRYPTODEV_ATTACHED;
 363                 cryptodev->pmd_type = type;
 364
 365                 cryptodev_globals.nb_devs++;
 366         }
 367
 368         return cryptodev;
 369 }
 370
 371 static inline int
 372 rte_cryptodev_create_unique_device_name(char *name, size_t size,
 373                 struct rte_pci_device *pci_dev)
 374 {
 375         int ret;
 376
 377         if ((name == NULL) || (pci_dev == NULL))
 378                 return -EINVAL;
 379
 380         ret = snprintf(name, size, "%d:%d.%d",
 381                         pci_dev->addr.bus, pci_dev->addr.devid,
 382                         pci_dev->addr.function);
 383         if (ret < 0)
 384                 return ret;
 385         return 0;
 386 }
 387
 388 int
 389 rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev)
 390 {
 391         int ret;
 392
 393         if (cryptodev == NULL)
 394                 return -EINVAL;
 395
 396         ret = rte_cryptodev_close(cryptodev->data->dev_id);
 397         if (ret < 0)
 398                 return ret;
 399
 400         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
 401         cryptodev_globals.nb_devs--;
 402         return 0;
 403 }
 404
 405 struct rte_cryptodev *
 406 rte_cryptodev_pmd_virtual_dev_init(const char *name, size_t dev_private_size,
 407                 int socket_id)
 408 {
 409         struct rte_cryptodev *cryptodev;
 410
 411         /* allocate device structure */
 412         cryptodev = rte_cryptodev_pmd_allocate(name, PMD_VDEV, socket_id);
 413         if (cryptodev == NULL)
 414                 return NULL;
 415
 416         /* allocate private device structure */
 417         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 418                 cryptodev->data->dev_private =
 419                                 rte_zmalloc_socket("cryptodev device private",
 420                                                 dev_private_size,
 421                                                 RTE_CACHE_LINE_SIZE,
 422                                                 socket_id);
 423
 424                 if (cryptodev->data->dev_private == NULL)
 425                         rte_panic("Cannot allocate memzone for private device"
 426                                         " data");
 427         }
 428
 429         /* initialise user call-back tail queue */
 430         TAILQ_INIT(&(cryptodev->link_intr_cbs));
 431
 432         return cryptodev;
 433 }
 434
 435 static int
 436 rte_cryptodev_init(struct rte_pci_driver *pci_drv,
 437                 struct rte_pci_device *pci_dev)
 438 {
 439         struct rte_cryptodev_driver *cryptodrv;
 440         struct rte_cryptodev *cryptodev;
 441
 442         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 443
 444         int retval;
 445
 446         cryptodrv = (struct rte_cryptodev_driver *)pci_drv;
 447         if (cryptodrv == NULL)
 448                 return -ENODEV;
 449
 450         /* Create unique Crypto device name using PCI address */
 451         rte_cryptodev_create_unique_device_name(cryptodev_name,
 452                         sizeof(cryptodev_name), pci_dev);
 453
 454         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, PMD_PDEV,
 455                         rte_socket_id());
 456         if (cryptodev == NULL)
 457                 return -ENOMEM;
 458
 459         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 460                 cryptodev->data->dev_private =
 461                                 rte_zmalloc_socket(
 462                                                 "cryptodev private structure",
 463                                                 cryptodrv->dev_private_size,
 464                                                 RTE_CACHE_LINE_SIZE,
 465                                                 rte_socket_id());
 466
 467                 if (cryptodev->data->dev_private == NULL)
 468                         rte_panic("Cannot allocate memzone for private "
 469                                         "device data");
 470         }
 471
 472         cryptodev->pci_dev = pci_dev;
 473         cryptodev->driver = cryptodrv;
 474
 475         /* init user callbacks */
 476         TAILQ_INIT(&(cryptodev->link_intr_cbs));
 477
 478         /* Invoke PMD device initialization function */
 479         retval = (*cryptodrv->cryptodev_init)(cryptodrv, cryptodev);
 480         if (retval == 0)
 481                 return 0;
 482
 483         CDEV_LOG_ERR("driver %s: crypto_dev_init(vendor_id=0x%x device_id=0x%x)"
 484                         " failed", pci_drv->name,
 485                         (unsigned) pci_dev->id.vendor_id,
 486                         (unsigned) pci_dev->id.device_id);
 487
 488         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 489                 rte_free(cryptodev->data->dev_private);
 490
 491         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
 492         cryptodev_globals.nb_devs--;
 493
 494         return -ENXIO;
 495 }
 496
 497 static int
 498 rte_cryptodev_uninit(struct rte_pci_device *pci_dev)
 499 {
 500         const struct rte_cryptodev_driver *cryptodrv;
 501         struct rte_cryptodev *cryptodev;
 502         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 503         int ret;
 504
 505         if (pci_dev == NULL)
 506                 return -EINVAL;
 507
 508         /* Create unique device name using PCI address */
 509         rte_cryptodev_create_unique_device_name(cryptodev_name,
 510                         sizeof(cryptodev_name), pci_dev);
 511
 512         cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
 513         if (cryptodev == NULL)
 514                 return -ENODEV;
 515
 516         cryptodrv = (const struct rte_cryptodev_driver *)pci_dev->driver;
 517         if (cryptodrv == NULL)
 518                 return -ENODEV;
 519
 520         /* Invoke PMD device uninit function */
 521         if (*cryptodrv->cryptodev_uninit) {
 522                 ret = (*cryptodrv->cryptodev_uninit)(cryptodrv, cryptodev);
 523                 if (ret)
 524                         return ret;
 525         }
 526
 527         /* free crypto device */
 528         rte_cryptodev_pmd_release_device(cryptodev);
 529
 530         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 531                 rte_free(cryptodev->data->dev_private);
 532
 533         cryptodev->pci_dev = NULL;
 534         cryptodev->driver = NULL;
 535         cryptodev->data = NULL;
 536
 537         return 0;
 538 }
 539
 540 int
 541 rte_cryptodev_pmd_driver_register(struct rte_cryptodev_driver *cryptodrv,
 542                 enum pmd_type type)
 543 {
 544         /* Call crypto device initialization directly if device is virtual */
 545         if (type == PMD_VDEV)
 546                 return rte_cryptodev_init((struct rte_pci_driver *)cryptodrv,
 547                                 NULL);
 548
 549         /*
 550          * Register PCI driver for physical device intialisation during
 551          * PCI probing
 552          */
 553         cryptodrv->pci_drv.devinit = rte_cryptodev_init;
 554         cryptodrv->pci_drv.devuninit = rte_cryptodev_uninit;
 555
 556         rte_eal_pci_register(&cryptodrv->pci_drv);
 557
 558         return 0;
 559 }
 560
 561
 562 uint16_t
 563 rte_cryptodev_queue_pair_count(uint8_t dev_id)
 564 {
 565         struct rte_cryptodev *dev;
 566
 567         dev = &rte_crypto_devices[dev_id];
 568         return dev->data->nb_queue_pairs;
 569 }
 570
 571 static int
 572 rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
 573                 int socket_id)
 574 {
 575         struct rte_cryptodev_info dev_info;
 576         void **qp;
 577         unsigned i;
 578
 579         if ((dev == NULL) || (nb_qpairs < 1)) {
 580                 CDEV_LOG_ERR("invalid param: dev %p, nb_queues %u",
 581                                                         dev, nb_qpairs);
 582                 return -EINVAL;
 583         }
 584
 585         CDEV_LOG_DEBUG("Setup %d queues pairs on device %u",
 586                         nb_qpairs, dev->data->dev_id);
 587
 588         memset(&dev_info, 0, sizeof(struct rte_cryptodev_info));
 589
 590         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
 591         (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
 592
 593         if (nb_qpairs > (dev_info.max_nb_queue_pairs)) {
 594                 CDEV_LOG_ERR("Invalid num queue_pairs (%u) for dev %u",
 595                                 nb_qpairs, dev->data->dev_id);
 596             return -EINVAL;
 597         }
 598
 599         if (dev->data->queue_pairs == NULL) { /* first time configuration */
 600                 dev->data->queue_pairs = rte_zmalloc_socket(
 601                                 "cryptodev->queue_pairs",
 602                                 sizeof(dev->data->queue_pairs[0]) * nb_qpairs,
 603                                 RTE_CACHE_LINE_SIZE, socket_id);
 604
 605                 if (dev->data->queue_pairs == NULL) {
 606                         dev->data->nb_queue_pairs = 0;
 607                         CDEV_LOG_ERR("failed to get memory for qp meta data, "
 608                                                         "nb_queues %u",
 609                                                         nb_qpairs);
 610                         return -(ENOMEM);
 611                 }
 612         } else { /* re-configure */
 613                 int ret;
 614                 uint16_t old_nb_queues = dev->data->nb_queue_pairs;
 615
 616                 qp = dev->data->queue_pairs;
 617
 618                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
 619                                 -ENOTSUP);
 620
 621                 for (i = nb_qpairs; i < old_nb_queues; i++) {
 622                         ret = (*dev->dev_ops->queue_pair_release)(dev, i);
 623                         if (ret < 0)
 624                                 return ret;
 625                 }
 626
 627                 qp = rte_realloc(qp, sizeof(qp[0]) * nb_qpairs,
 628                                 RTE_CACHE_LINE_SIZE);
 629                 if (qp == NULL) {
 630                         CDEV_LOG_ERR("failed to realloc qp meta data,"
 631                                                 " nb_queues %u", nb_qpairs);
 632                         return -(ENOMEM);
 633                 }
 634
 635                 if (nb_qpairs > old_nb_queues) {
 636                         uint16_t new_qs = nb_qpairs - old_nb_queues;
 637
 638                         memset(qp + old_nb_queues, 0,
 639                                 sizeof(qp[0]) * new_qs);
 640                 }
 641
 642                 dev->data->queue_pairs = qp;
 643
 644         }
 645         dev->data->nb_queue_pairs = nb_qpairs;
 646         return 0;
 647 }
 648
 649 int
 650 rte_cryptodev_queue_pair_start(uint8_t dev_id, uint16_t queue_pair_id)
 651 {
 652         struct rte_cryptodev *dev;
 653
 654         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 655                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 656                 return -EINVAL;
 657         }
 658
 659         dev = &rte_crypto_devices[dev_id];
 660         if (queue_pair_id >= dev->data->nb_queue_pairs) {
 661                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
 662                 return -EINVAL;
 663         }
 664
 665         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_start, -ENOTSUP);
 666
 667         return dev->dev_ops->queue_pair_start(dev, queue_pair_id);
 668
 669 }
 670
 671 int
 672 rte_cryptodev_queue_pair_stop(uint8_t dev_id, uint16_t queue_pair_id)
 673 {
 674         struct rte_cryptodev *dev;
 675
 676         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 677                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 678                 return -EINVAL;
 679         }
 680
 681         dev = &rte_crypto_devices[dev_id];
 682         if (queue_pair_id >= dev->data->nb_queue_pairs) {
 683                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
 684                 return -EINVAL;
 685         }
 686
 687         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_stop, -ENOTSUP);
 688
 689         return dev->dev_ops->queue_pair_stop(dev, queue_pair_id);
 690
 691 }
 692
 693 static int
 694 rte_cryptodev_sym_session_pool_create(struct rte_cryptodev *dev,
 695                 unsigned nb_objs, unsigned obj_cache_size, int socket_id);
 696
 697 int
 698 rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config)
 699 {
 700         struct rte_cryptodev *dev;
 701         int diag;
 702
 703         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 704                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 705                 return -EINVAL;
 706         }
 707
 708         dev = &rte_crypto_devices[dev_id];
 709
 710         if (dev->data->dev_started) {
 711                 CDEV_LOG_ERR(
 712                     "device %d must be stopped to allow configuration", dev_id);
 713                 return -EBUSY;
 714         }
 715
 716         /* Setup new number of queue pairs and reconfigure device. */
 717         diag = rte_cryptodev_queue_pairs_config(dev, config->nb_queue_pairs,
 718                         config->socket_id);
 719         if (diag != 0) {
 720                 CDEV_LOG_ERR("dev%d rte_crypto_dev_queue_pairs_config = %d",
 721                                 dev_id, diag);
 722                 return diag;
 723         }
 724
 725         /* Setup Session mempool for device */
 726         return rte_cryptodev_sym_session_pool_create(dev,
 727                         config->session_mp.nb_objs,
 728                         config->session_mp.cache_size,
 729                         config->socket_id);
 730 }
 731
 732
 733 int
 734 rte_cryptodev_start(uint8_t dev_id)
 735 {
 736         struct rte_cryptodev *dev;
 737         int diag;
 738
 739         CDEV_LOG_DEBUG("Start dev_id=%" PRIu8, dev_id);
 740
 741         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 742                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 743                 return -EINVAL;
 744         }
 745
 746         dev = &rte_crypto_devices[dev_id];
 747
 748         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
 749
 750         if (dev->data->dev_started != 0) {
 751                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already started",
 752                         dev_id);
 753                 return 0;
 754         }
 755
 756         diag = (*dev->dev_ops->dev_start)(dev);
 757         if (diag == 0)
 758                 dev->data->dev_started = 1;
 759         else
 760                 return diag;
 761
 762         return 0;
 763 }
 764
 765 void
 766 rte_cryptodev_stop(uint8_t dev_id)
 767 {
 768         struct rte_cryptodev *dev;
 769
 770         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 771                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 772                 return;
 773         }
 774
 775         dev = &rte_crypto_devices[dev_id];
 776
 777         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
 778
 779         if (dev->data->dev_started == 0) {
 780                 CDEV_LOG_ERR("Device with dev_id=%" PRIu8 " already stopped",
 781                         dev_id);
 782                 return;
 783         }
 784
 785         dev->data->dev_started = 0;
 786         (*dev->dev_ops->dev_stop)(dev);
 787 }
 788
 789 int
 790 rte_cryptodev_close(uint8_t dev_id)
 791 {
 792         struct rte_cryptodev *dev;
 793         int retval;
 794
 795         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 796                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 797                 return -1;
 798         }
 799
 800         dev = &rte_crypto_devices[dev_id];
 801
 802         /* Device must be stopped before it can be closed */
 803         if (dev->data->dev_started == 1) {
 804                 CDEV_LOG_ERR("Device %u must be stopped before closing",
 805                                 dev_id);
 806                 return -EBUSY;
 807         }
 808
 809         /* We can't close the device if there are outstanding sessions in use */
 810         if (dev->data->session_pool != NULL) {
 811                 if (!rte_mempool_full(dev->data->session_pool)) {
 812                         CDEV_LOG_ERR("dev_id=%u close failed, session mempool "
 813                                         "has sessions still in use, free "
 814                                         "all sessions before calling close",
 815                                         (unsigned)dev_id);
 816                         return -EBUSY;
 817                 }
 818         }
 819
 820         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
 821         retval = (*dev->dev_ops->dev_close)(dev);
 822
 823         if (retval < 0)
 824                 return retval;
 825
 826         return 0;
 827 }
 828
 829 int
 830 rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
 831                 const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
 832 {
 833         struct rte_cryptodev *dev;
 834
 835         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 836                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 837                 return -EINVAL;
 838         }
 839
 840         dev = &rte_crypto_devices[dev_id];
 841         if (queue_pair_id >= dev->data->nb_queue_pairs) {
 842                 CDEV_LOG_ERR("Invalid queue_pair_id=%d", queue_pair_id);
 843                 return -EINVAL;
 844         }
 845
 846         if (dev->data->dev_started) {
 847                 CDEV_LOG_ERR(
 848                     "device %d must be stopped to allow configuration", dev_id);
 849                 return -EBUSY;
 850         }
 851
 852         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP);
 853
 854         return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id, qp_conf,
 855                         socket_id);
 856 }
 857
 858
 859 int
 860 rte_cryptodev_stats_get(uint8_t dev_id, struct rte_cryptodev_stats *stats)
 861 {
 862         struct rte_cryptodev *dev;
 863
 864         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 865                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
 866                 return -ENODEV;
 867         }
 868
 869         if (stats == NULL) {
 870                 CDEV_LOG_ERR("Invalid stats ptr");
 871                 return -EINVAL;
 872         }
 873
 874         dev = &rte_crypto_devices[dev_id];
 875         memset(stats, 0, sizeof(*stats));
 876
 877         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
 878         (*dev->dev_ops->stats_get)(dev, stats);
 879         return 0;
 880 }
 881
 882 void
 883 rte_cryptodev_stats_reset(uint8_t dev_id)
 884 {
 885         struct rte_cryptodev *dev;
 886
 887         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 888                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 889                 return;
 890         }
 891
 892         dev = &rte_crypto_devices[dev_id];
 893
 894         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
 895         (*dev->dev_ops->stats_reset)(dev);
 896 }
 897
 898
 899 void
 900 rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info)
 901 {
 902         struct rte_cryptodev *dev;
 903
 904         if (dev_id >= cryptodev_globals.nb_devs) {
 905                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
 906                 return;
 907         }
 908
 909         dev = &rte_crypto_devices[dev_id];
 910
 911         memset(dev_info, 0, sizeof(struct rte_cryptodev_info));
 912
 913         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
 914         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
 915
 916         dev_info->pci_dev = dev->pci_dev;
 917         if (dev->driver)
 918                 dev_info->driver_name = dev->driver->pci_drv.name;
 919 }
 920
 921
 922 int
 923 rte_cryptodev_callback_register(uint8_t dev_id,
 924                         enum rte_cryptodev_event_type event,
 925                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
 926 {
 927         struct rte_cryptodev *dev;
 928         struct rte_cryptodev_callback *user_cb;
 929
 930         if (!cb_fn)
 931                 return -EINVAL;
 932
 933         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 934                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 935                 return -EINVAL;
 936         }
 937
 938         dev = &rte_crypto_devices[dev_id];
 939         rte_spinlock_lock(&rte_cryptodev_cb_lock);
 940
 941         TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
 942                 if (user_cb->cb_fn == cb_fn &&
 943                         user_cb->cb_arg == cb_arg &&
 944                         user_cb->event == event) {
 945                         break;
 946                 }
 947         }
 948
 949         /* create a new callback. */
 950         if (user_cb == NULL) {
 951                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
 952                                 sizeof(struct rte_cryptodev_callback), 0);
 953                 if (user_cb != NULL) {
 954                         user_cb->cb_fn = cb_fn;
 955                         user_cb->cb_arg = cb_arg;
 956                         user_cb->event = event;
 957                         TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
 958                 }
 959         }
 960
 961         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
 962         return (user_cb == NULL) ? -ENOMEM : 0;
 963 }
 964
 965 int
 966 rte_cryptodev_callback_unregister(uint8_t dev_id,
 967                         enum rte_cryptodev_event_type event,
 968                         rte_cryptodev_cb_fn cb_fn, void *cb_arg)
 969 {
 970         int ret;
 971         struct rte_cryptodev *dev;
 972         struct rte_cryptodev_callback *cb, *next;
 973
 974         if (!cb_fn)
 975                 return -EINVAL;
 976
 977         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
 978                 CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
 979                 return -EINVAL;
 980         }
 981
 982         dev = &rte_crypto_devices[dev_id];
 983         rte_spinlock_lock(&rte_cryptodev_cb_lock);
 984
 985         ret = 0;
 986         for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
 987
 988                 next = TAILQ_NEXT(cb, next);
 989
 990                 if (cb->cb_fn != cb_fn || cb->event != event ||
 991                                 (cb->cb_arg != (void *)-1 &&
 992                                 cb->cb_arg != cb_arg))
 993                         continue;
 994
 995                 /*
 996                  * if this callback is not executing right now,
 997                  * then remove it.
 998                  */
 999                 if (cb->active == 0) {
1000                         TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
1001                         rte_free(cb);
1002                 } else {
1003                         ret = -EAGAIN;
1004                 }
1005         }
1006
1007         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1008         return ret;
1009 }
1010
1011 void
1012 rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev,
1013         enum rte_cryptodev_event_type event)
1014 {
1015         struct rte_cryptodev_callback *cb_lst;
1016         struct rte_cryptodev_callback dev_cb;
1017
1018         rte_spinlock_lock(&rte_cryptodev_cb_lock);
1019         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
1020                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
1021                         continue;
1022                 dev_cb = *cb_lst;
1023                 cb_lst->active = 1;
1024                 rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1025                 dev_cb.cb_fn(dev->data->dev_id, dev_cb.event,
1026                                                 dev_cb.cb_arg);
1027                 rte_spinlock_lock(&rte_cryptodev_cb_lock);
1028                 cb_lst->active = 0;
1029         }
1030         rte_spinlock_unlock(&rte_cryptodev_cb_lock);
1031 }
1032
1033
1034 static void
1035 rte_cryptodev_sym_session_init(struct rte_mempool *mp,
1036                 void *opaque_arg,
1037                 void *_sess,
1038                 __rte_unused unsigned i)
1039 {
1040         struct rte_cryptodev_sym_session *sess = _sess;
1041         struct rte_cryptodev *dev = opaque_arg;
1042
1043         memset(sess, 0, mp->elt_size);
1044
1045         sess->dev_id = dev->data->dev_id;
1046         sess->dev_type = dev->dev_type;
1047         sess->mp = mp;
1048
1049         if (dev->dev_ops->session_initialize)
1050                 (*dev->dev_ops->session_initialize)(mp, sess);
1051 }
1052
1053 static int
1054 rte_cryptodev_sym_session_pool_create(struct rte_cryptodev *dev,
1055                 unsigned nb_objs, unsigned obj_cache_size, int socket_id)
1056 {
1057         char mp_name[RTE_CRYPTODEV_NAME_MAX_LEN];
1058         unsigned priv_sess_size;
1059
1060         unsigned n = snprintf(mp_name, sizeof(mp_name), "cdev_%d_sess_mp",
1061                         dev->data->dev_id);
1062         if (n > sizeof(mp_name)) {
1063                 CDEV_LOG_ERR("Unable to create unique name for session mempool");
1064                 return -ENOMEM;
1065         }
1066
1067         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_get_size, -ENOTSUP);
1068         priv_sess_size = (*dev->dev_ops->session_get_size)(dev);
1069         if (priv_sess_size == 0) {
1070                 CDEV_LOG_ERR("%s returned and invalid private session size ",
1071                                                 dev->data->name);
1072                 return -ENOMEM;
1073         }
1074
1075         unsigned elt_size = sizeof(struct rte_cryptodev_sym_session) +
1076                         priv_sess_size;
1077
1078         dev->data->session_pool = rte_mempool_lookup(mp_name);
1079         if (dev->data->session_pool != NULL) {
1080                 if ((dev->data->session_pool->elt_size != elt_size) ||
1081                                 (dev->data->session_pool->cache_size <
1082                                 obj_cache_size) ||
1083                                 (dev->data->session_pool->size < nb_objs)) {
1084
1085                         CDEV_LOG_ERR("%s mempool already exists with different"
1086                                         " initialization parameters", mp_name);
1087                         dev->data->session_pool = NULL;
1088                         return -ENOMEM;
1089                 }
1090         } else {
1091                 dev->data->session_pool = rte_mempool_create(
1092                                 mp_name, /* mempool name */
1093                                 nb_objs, /* number of elements*/
1094                                 elt_size, /* element size*/
1095                                 obj_cache_size, /* Cache size*/
1096                                 0, /* private data size */
1097                                 NULL, /* obj initialization constructor */
1098                                 NULL, /* obj initialization constructor arg */
1099                                 rte_cryptodev_sym_session_init,
1100                                 /**< obj constructor*/
1101                                 dev, /* obj constructor arg */
1102                                 socket_id, /* socket id */
1103                                 0); /* flags */
1104
1105                 if (dev->data->session_pool == NULL) {
1106                         CDEV_LOG_ERR("%s mempool allocation failed", mp_name);
1107                         return -ENOMEM;
1108                 }
1109         }
1110
1111         CDEV_LOG_DEBUG("%s mempool created!", mp_name);
1112         return 0;
1113 }
1114
1115 struct rte_cryptodev_sym_session *
1116 rte_cryptodev_sym_session_create(uint8_t dev_id,
1117                 struct rte_crypto_sym_xform *xform)
1118 {
1119         struct rte_cryptodev *dev;
1120         struct rte_cryptodev_sym_session *sess;
1121         void *_sess;
1122
1123         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1124                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1125                 return NULL;
1126         }
1127
1128         dev = &rte_crypto_devices[dev_id];
1129
1130         /* Allocate a session structure from the session pool */
1131         if (rte_mempool_get(dev->data->session_pool, &_sess)) {
1132                 CDEV_LOG_ERR("Couldn't get object from session mempool");
1133                 return NULL;
1134         }
1135
1136         sess = (struct rte_cryptodev_sym_session *)_sess;
1137
1138         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_configure, NULL);
1139         if (dev->dev_ops->session_configure(dev, xform, sess->_private) ==
1140                         NULL) {
1141                 CDEV_LOG_ERR("dev_id %d failed to configure session details",
1142                                 dev_id);
1143
1144                 /* Return session to mempool */
1145                 rte_mempool_put(sess->mp, _sess);
1146                 return NULL;
1147         }
1148
1149         return sess;
1150 }
1151
1152 struct rte_cryptodev_sym_session *
1153 rte_cryptodev_sym_session_free(uint8_t dev_id,
1154                 struct rte_cryptodev_sym_session *sess)
1155 {
1156         struct rte_cryptodev *dev;
1157
1158         if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
1159                 CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
1160                 return sess;
1161         }
1162
1163         dev = &rte_crypto_devices[dev_id];
1164
1165         /* Check the session belongs to this device type */
1166         if (sess->dev_type != dev->dev_type)
1167                 return sess;
1168
1169         /* Let device implementation clear session material */
1170         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_clear, sess);
1171         dev->dev_ops->session_clear(dev, (void *)sess->_private);
1172
1173         /* Return session to mempool */
1174         rte_mempool_put(sess->mp, (void *)sess);
1175
1176         return NULL;
1177 }
1178
1179 /** Initialise rte_crypto_op mempool element */
1180 static void
1181 rte_crypto_op_init(struct rte_mempool *mempool,
1182                 void *opaque_arg,
1183                 void *_op_data,
1184                 __rte_unused unsigned i)
1185 {
1186         struct rte_crypto_op *op = _op_data;
1187         enum rte_crypto_op_type type = *(enum rte_crypto_op_type *)opaque_arg;
1188
1189         memset(_op_data, 0, mempool->elt_size);
1190
1191         __rte_crypto_op_reset(op, type);
1192
1193         op->phys_addr = rte_mem_virt2phy(_op_data);
1194         op->mempool = mempool;
1195 }
1196
1197
1198 struct rte_mempool *
1199 rte_crypto_op_pool_create(const char *name, enum rte_crypto_op_type type,
1200                 unsigned nb_elts, unsigned cache_size, uint16_t priv_size,
1201                 int socket_id)
1202 {
1203         struct rte_crypto_op_pool_private *priv;
1204
1205         unsigned elt_size = sizeof(struct rte_crypto_op) +
1206                         sizeof(struct rte_crypto_sym_op) +
1207                         priv_size;
1208
1209         /* lookup mempool in case already allocated */
1210         struct rte_mempool *mp = rte_mempool_lookup(name);
1211
1212         if (mp != NULL) {
1213                 priv = (struct rte_crypto_op_pool_private *)
1214                                 rte_mempool_get_priv(mp);
1215
1216                 if (mp->elt_size != elt_size ||
1217                                 mp->cache_size < cache_size ||
1218                                 mp->size < nb_elts ||
1219                                 priv->priv_size <  priv_size) {
1220                         mp = NULL;
1221                         CDEV_LOG_ERR("Mempool %s already exists but with "
1222                                         "incompatible parameters", name);
1223                         return NULL;
1224                 }
1225                 return mp;
1226         }
1227
1228         mp = rte_mempool_create(
1229                         name,
1230                         nb_elts,
1231                         elt_size,
1232                         cache_size,
1233                         sizeof(struct rte_crypto_op_pool_private),
1234                         NULL,
1235                         NULL,
1236                         rte_crypto_op_init,
1237                         &type,
1238                         socket_id,
1239                         0);
1240
1241         if (mp == NULL) {
1242                 CDEV_LOG_ERR("Failed to create mempool %s", name);
1243                 return NULL;
1244         }
1245
1246         priv = (struct rte_crypto_op_pool_private *)
1247                         rte_mempool_get_priv(mp);
1248
1249         priv->priv_size = priv_size;
1250         priv->type = type;
1251
1252         return mp;
1253 }