lib/librte_eventdev/rte_event_crypto_adapter.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation.
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <stdbool.h>
   8 #include <rte_common.h>
   9 #include <rte_dev.h>
  10 #include <rte_errno.h>
  11 #include <rte_cryptodev.h>
  12 #include <rte_cryptodev_pmd.h>
  13 #include <rte_log.h>
  14 #include <rte_malloc.h>
  15 #include <rte_service_component.h>
  16
  17 #include "rte_eventdev.h"
  18 #include "rte_eventdev_pmd.h"
  19 #include "rte_event_crypto_adapter.h"
  20
  21 #define BATCH_SIZE 32
  22 #define DEFAULT_MAX_NB 128
  23 #define CRYPTO_ADAPTER_NAME_LEN 32
  24 #define CRYPTO_ADAPTER_MEM_NAME_LEN 32
  25 #define CRYPTO_ADAPTER_MAX_EV_ENQ_RETRIES 100
  26
  27 /* Flush an instance's enqueue buffers every CRYPTO_ENQ_FLUSH_THRESHOLD
  28  * iterations of eca_crypto_adapter_enq_run()
  29  */
  30 #define CRYPTO_ENQ_FLUSH_THRESHOLD 1024
  31
  32 struct rte_event_crypto_adapter {
  33         /* Event device identifier */
  34         uint8_t eventdev_id;
  35         /* Event port identifier */
  36         uint8_t event_port_id;
  37         /* Store event device's implicit release capability */
  38         uint8_t implicit_release_disabled;
  39         /* Max crypto ops processed in any service function invocation */
  40         uint32_t max_nb;
  41         /* Lock to serialize config updates with service function */
  42         rte_spinlock_t lock;
  43         /* Next crypto device to be processed */
  44         uint16_t next_cdev_id;
  45         /* Per crypto device structure */
  46         struct crypto_device_info *cdevs;
  47         /* Loop counter to flush crypto ops */
  48         uint16_t transmit_loop_count;
  49         /* Per instance stats structure */
  50         struct rte_event_crypto_adapter_stats crypto_stats;
  51         /* Configuration callback for rte_service configuration */
  52         rte_event_crypto_adapter_conf_cb conf_cb;
  53         /* Configuration callback argument */
  54         void *conf_arg;
  55         /* Set if  default_cb is being used */
  56         int default_cb_arg;
  57         /* Service initialization state */
  58         uint8_t service_inited;
  59         /* Memory allocation name */
  60         char mem_name[CRYPTO_ADAPTER_MEM_NAME_LEN];
  61         /* Socket identifier cached from eventdev */
  62         int socket_id;
  63         /* Per adapter EAL service */
  64         uint32_t service_id;
  65         /* No. of queue pairs configured */
  66         uint16_t nb_qps;
  67         /* Adapter mode */
  68         enum rte_event_crypto_adapter_mode mode;
  69 } __rte_cache_aligned;
  70
  71 /* Per crypto device information */
  72 struct crypto_device_info {
  73         /* Pointer to cryptodev */
  74         struct rte_cryptodev *dev;
  75         /* Pointer to queue pair info */
  76         struct crypto_queue_pair_info *qpairs;
  77         /* Next queue pair to be processed */
  78         uint16_t next_queue_pair_id;
  79         /* Set to indicate cryptodev->eventdev packet
  80          * transfer uses a hardware mechanism
  81          */
  82         uint8_t internal_event_port;
  83         /* Set to indicate processing has been started */
  84         uint8_t dev_started;
  85         /* If num_qpairs > 0, the start callback will
  86          * be invoked if not already invoked
  87          */
  88         uint16_t num_qpairs;
  89 } __rte_cache_aligned;
  90
  91 /* Per queue pair information */
  92 struct crypto_queue_pair_info {
  93         /* Set to indicate queue pair is enabled */
  94         bool qp_enabled;
  95         /* Pointer to hold rte_crypto_ops for batching */
  96         struct rte_crypto_op **op_buffer;
  97         /* No of crypto ops accumulated */
  98         uint8_t len;
  99 } __rte_cache_aligned;
 100
 101 static struct rte_event_crypto_adapter **event_crypto_adapter;
 102
 103 /* Macros to check for valid adapter */
 104 #define EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \
 105         if (!eca_valid_id(id)) { \
 106                 RTE_EDEV_LOG_ERR("Invalid crypto adapter id = %d\n", id); \
 107                 return retval; \
 108         } \
 109 } while (0)
 110
 111 static inline int
 112 eca_valid_id(uint8_t id)
 113 {
 114         return id < RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE;
 115 }
 116
 117 static int
 118 eca_init(void)
 119 {
 120         const char *name = "crypto_adapter_array";
 121         const struct rte_memzone *mz;
 122         unsigned int sz;
 123
 124         sz = sizeof(*event_crypto_adapter) *
 125             RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE;
 126         sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE);
 127
 128         mz = rte_memzone_lookup(name);
 129         if (mz == NULL) {
 130                 mz = rte_memzone_reserve_aligned(name, sz, rte_socket_id(), 0,
 131                                                  RTE_CACHE_LINE_SIZE);
 132                 if (mz == NULL) {
 133                         RTE_EDEV_LOG_ERR("failed to reserve memzone err = %"
 134                                         PRId32, rte_errno);
 135                         return -rte_errno;
 136                 }
 137         }
 138
 139         event_crypto_adapter = mz->addr;
 140         return 0;
 141 }
 142
 143 static inline struct rte_event_crypto_adapter *
 144 eca_id_to_adapter(uint8_t id)
 145 {
 146         return event_crypto_adapter ?
 147                 event_crypto_adapter[id] : NULL;
 148 }
 149
 150 static int
 151 eca_default_config_cb(uint8_t id, uint8_t dev_id,
 152                         struct rte_event_crypto_adapter_conf *conf, void *arg)
 153 {
 154         struct rte_event_dev_config dev_conf;
 155         struct rte_eventdev *dev;
 156         uint8_t port_id;
 157         int started;
 158         int ret;
 159         struct rte_event_port_conf *port_conf = arg;
 160         struct rte_event_crypto_adapter *adapter = eca_id_to_adapter(id);
 161
 162         if (adapter == NULL)
 163                 return -EINVAL;
 164
 165         dev = &rte_eventdevs[adapter->eventdev_id];
 166         dev_conf = dev->data->dev_conf;
 167
 168         started = dev->data->dev_started;
 169         if (started)
 170                 rte_event_dev_stop(dev_id);
 171         port_id = dev_conf.nb_event_ports;
 172         dev_conf.nb_event_ports += 1;
 173         ret = rte_event_dev_configure(dev_id, &dev_conf);
 174         if (ret) {
 175                 RTE_EDEV_LOG_ERR("failed to configure event dev %u\n", dev_id);
 176                 if (started) {
 177                         if (rte_event_dev_start(dev_id))
 178                                 return -EIO;
 179                 }
 180                 return ret;
 181         }
 182
 183         ret = rte_event_port_setup(dev_id, port_id, port_conf);
 184         if (ret) {
 185                 RTE_EDEV_LOG_ERR("failed to setup event port %u\n", port_id);
 186                 return ret;
 187         }
 188
 189         conf->event_port_id = port_id;
 190         conf->max_nb = DEFAULT_MAX_NB;
 191         if (started)
 192                 ret = rte_event_dev_start(dev_id);
 193
 194         adapter->default_cb_arg = 1;
 195         return ret;
 196 }
 197
 198 int __rte_experimental
 199 rte_event_crypto_adapter_create_ext(uint8_t id, uint8_t dev_id,
 200                                 rte_event_crypto_adapter_conf_cb conf_cb,
 201                                 enum rte_event_crypto_adapter_mode mode,
 202                                 void *conf_arg)
 203 {
 204         struct rte_event_crypto_adapter *adapter;
 205         char mem_name[CRYPTO_ADAPTER_NAME_LEN];
 206         struct rte_event_dev_info dev_info;
 207         int socket_id;
 208         uint8_t i;
 209         int ret;
 210
 211         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 212         RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
 213         if (conf_cb == NULL)
 214                 return -EINVAL;
 215
 216         if (event_crypto_adapter == NULL) {
 217                 ret = eca_init();
 218                 if (ret)
 219                         return ret;
 220         }
 221
 222         adapter = eca_id_to_adapter(id);
 223         if (adapter != NULL) {
 224                 RTE_EDEV_LOG_ERR("Crypto adapter id %u already exists!", id);
 225                 return -EEXIST;
 226         }
 227
 228         socket_id = rte_event_dev_socket_id(dev_id);
 229         snprintf(mem_name, CRYPTO_ADAPTER_MEM_NAME_LEN,
 230                  "rte_event_crypto_adapter_%d", id);
 231
 232         adapter = rte_zmalloc_socket(mem_name, sizeof(*adapter),
 233                         RTE_CACHE_LINE_SIZE, socket_id);
 234         if (adapter == NULL) {
 235                 RTE_EDEV_LOG_ERR("Failed to get mem for event crypto adapter!");
 236                 return -ENOMEM;
 237         }
 238
 239         ret = rte_event_dev_info_get(dev_id, &dev_info);
 240         if (ret < 0) {
 241                 RTE_EDEV_LOG_ERR("Failed to get info for eventdev %d: %s!",
 242                                  dev_id, dev_info.driver_name);
 243                 return ret;
 244         }
 245
 246         adapter->implicit_release_disabled = (dev_info.event_dev_cap &
 247                         RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE);
 248         adapter->eventdev_id = dev_id;
 249         adapter->socket_id = socket_id;
 250         adapter->conf_cb = conf_cb;
 251         adapter->conf_arg = conf_arg;
 252         adapter->mode = mode;
 253         strcpy(adapter->mem_name, mem_name);
 254         adapter->cdevs = rte_zmalloc_socket(adapter->mem_name,
 255                                         rte_cryptodev_count() *
 256                                         sizeof(struct crypto_device_info), 0,
 257                                         socket_id);
 258         if (adapter->cdevs == NULL) {
 259                 RTE_EDEV_LOG_ERR("Failed to get mem for crypto devices\n");
 260                 rte_free(adapter);
 261                 return -ENOMEM;
 262         }
 263
 264         rte_spinlock_init(&adapter->lock);
 265         for (i = 0; i < rte_cryptodev_count(); i++)
 266                 adapter->cdevs[i].dev = rte_cryptodev_pmd_get_dev(i);
 267
 268         event_crypto_adapter[id] = adapter;
 269
 270         return 0;
 271 }
 272
 273
 274 int __rte_experimental
 275 rte_event_crypto_adapter_create(uint8_t id, uint8_t dev_id,
 276                                 struct rte_event_port_conf *port_config,
 277                                 enum rte_event_crypto_adapter_mode mode)
 278 {
 279         struct rte_event_port_conf *pc;
 280         int ret;
 281
 282         if (port_config == NULL)
 283                 return -EINVAL;
 284         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 285
 286         pc = rte_malloc(NULL, sizeof(*pc), 0);
 287         if (pc == NULL)
 288                 return -ENOMEM;
 289         *pc = *port_config;
 290         ret = rte_event_crypto_adapter_create_ext(id, dev_id,
 291                                                   eca_default_config_cb,
 292                                                   mode,
 293                                                   pc);
 294         if (ret)
 295                 rte_free(pc);
 296
 297         return ret;
 298 }
 299
 300 int __rte_experimental
 301 rte_event_crypto_adapter_free(uint8_t id)
 302 {
 303         struct rte_event_crypto_adapter *adapter;
 304
 305         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 306
 307         adapter = eca_id_to_adapter(id);
 308         if (adapter == NULL)
 309                 return -EINVAL;
 310
 311         if (adapter->nb_qps) {
 312                 RTE_EDEV_LOG_ERR("%" PRIu16 "Queue pairs not deleted",
 313                                 adapter->nb_qps);
 314                 return -EBUSY;
 315         }
 316
 317         if (adapter->default_cb_arg)
 318                 rte_free(adapter->conf_arg);
 319         rte_free(adapter->cdevs);
 320         rte_free(adapter);
 321         event_crypto_adapter[id] = NULL;
 322
 323         return 0;
 324 }
 325
 326 static inline unsigned int
 327 eca_enq_to_cryptodev(struct rte_event_crypto_adapter *adapter,
 328                  struct rte_event *ev, unsigned int cnt)
 329 {
 330         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 331         union rte_event_crypto_metadata *m_data = NULL;
 332         struct crypto_queue_pair_info *qp_info = NULL;
 333         struct rte_crypto_op *crypto_op;
 334         unsigned int i, n;
 335         uint16_t qp_id, len, ret;
 336         uint8_t cdev_id;
 337
 338         len = 0;
 339         ret = 0;
 340         n = 0;
 341         stats->event_deq_count += cnt;
 342
 343         for (i = 0; i < cnt; i++) {
 344                 crypto_op = ev[i].event_ptr;
 345                 if (crypto_op == NULL)
 346                         continue;
 347                 if (crypto_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 348                         m_data = rte_cryptodev_sym_session_get_user_data(
 349                                         crypto_op->sym->session);
 350                         if (m_data == NULL) {
 351                                 rte_pktmbuf_free(crypto_op->sym->m_src);
 352                                 rte_crypto_op_free(crypto_op);
 353                                 continue;
 354                         }
 355
 356                         cdev_id = m_data->request_info.cdev_id;
 357                         qp_id = m_data->request_info.queue_pair_id;
 358                         qp_info = &adapter->cdevs[cdev_id].qpairs[qp_id];
 359                         if (!qp_info->qp_enabled) {
 360                                 rte_pktmbuf_free(crypto_op->sym->m_src);
 361                                 rte_crypto_op_free(crypto_op);
 362                                 continue;
 363                         }
 364                         len = qp_info->len;
 365                         qp_info->op_buffer[len] = crypto_op;
 366                         len++;
 367                 } else if (crypto_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS &&
 368                                 crypto_op->private_data_offset) {
 369                         m_data = (union rte_event_crypto_metadata *)
 370                                  ((uint8_t *)crypto_op +
 371                                         crypto_op->private_data_offset);
 372                         cdev_id = m_data->request_info.cdev_id;
 373                         qp_id = m_data->request_info.queue_pair_id;
 374                         qp_info = &adapter->cdevs[cdev_id].qpairs[qp_id];
 375                         if (!qp_info->qp_enabled) {
 376                                 rte_pktmbuf_free(crypto_op->sym->m_src);
 377                                 rte_crypto_op_free(crypto_op);
 378                                 continue;
 379                         }
 380                         len = qp_info->len;
 381                         qp_info->op_buffer[len] = crypto_op;
 382                         len++;
 383                 } else {
 384                         rte_pktmbuf_free(crypto_op->sym->m_src);
 385                         rte_crypto_op_free(crypto_op);
 386                         continue;
 387                 }
 388
 389                 if (len == BATCH_SIZE) {
 390                         struct rte_crypto_op **op_buffer = qp_info->op_buffer;
 391                         ret = rte_cryptodev_enqueue_burst(cdev_id,
 392                                                           qp_id,
 393                                                           op_buffer,
 394                                                           BATCH_SIZE);
 395
 396                         stats->crypto_enq_count += ret;
 397
 398                         while (ret < len) {
 399                                 struct rte_crypto_op *op;
 400                                 op = op_buffer[ret++];
 401                                 stats->crypto_enq_fail++;
 402                                 rte_pktmbuf_free(op->sym->m_src);
 403                                 rte_crypto_op_free(op);
 404                         }
 405
 406                         len = 0;
 407                 }
 408
 409                 if (qp_info)
 410                         qp_info->len = len;
 411                 n += ret;
 412         }
 413
 414         return n;
 415 }
 416
 417 static unsigned int
 418 eca_crypto_enq_flush(struct rte_event_crypto_adapter *adapter)
 419 {
 420         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 421         struct crypto_device_info *curr_dev;
 422         struct crypto_queue_pair_info *curr_queue;
 423         struct rte_crypto_op **op_buffer;
 424         struct rte_cryptodev *dev;
 425         uint8_t cdev_id;
 426         uint16_t qp;
 427         uint16_t ret;
 428         uint16_t num_cdev = rte_cryptodev_count();
 429
 430         ret = 0;
 431         for (cdev_id = 0; cdev_id < num_cdev; cdev_id++) {
 432                 curr_dev = &adapter->cdevs[cdev_id];
 433                 dev = curr_dev->dev;
 434                 if (dev == NULL)
 435                         continue;
 436                 for (qp = 0; qp < dev->data->nb_queue_pairs; qp++) {
 437
 438                         curr_queue = &curr_dev->qpairs[qp];
 439                         if (!curr_queue->qp_enabled)
 440                                 continue;
 441
 442                         op_buffer = curr_queue->op_buffer;
 443                         ret = rte_cryptodev_enqueue_burst(cdev_id,
 444                                                           qp,
 445                                                           op_buffer,
 446                                                           curr_queue->len);
 447                         stats->crypto_enq_count += ret;
 448
 449                         while (ret < curr_queue->len) {
 450                                 struct rte_crypto_op *op;
 451                                 op = op_buffer[ret++];
 452                                 stats->crypto_enq_fail++;
 453                                 rte_pktmbuf_free(op->sym->m_src);
 454                                 rte_crypto_op_free(op);
 455                         }
 456                         curr_queue->len = 0;
 457                 }
 458         }
 459
 460         return ret;
 461 }
 462
 463 static int
 464 eca_crypto_adapter_enq_run(struct rte_event_crypto_adapter *adapter,
 465                         unsigned int max_enq)
 466 {
 467         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 468         struct rte_event ev[BATCH_SIZE];
 469         unsigned int nb_enq, nb_enqueued;
 470         uint16_t n;
 471         uint8_t event_dev_id = adapter->eventdev_id;
 472         uint8_t event_port_id = adapter->event_port_id;
 473
 474         nb_enqueued = 0;
 475         if (adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)
 476                 return 0;
 477
 478         for (nb_enq = 0; nb_enq < max_enq; nb_enq += n) {
 479                 stats->event_poll_count++;
 480                 n = rte_event_dequeue_burst(event_dev_id,
 481                                             event_port_id, ev, BATCH_SIZE, 0);
 482
 483                 if (!n)
 484                         break;
 485
 486                 nb_enqueued += eca_enq_to_cryptodev(adapter, ev, n);
 487         }
 488
 489         if ((++adapter->transmit_loop_count &
 490                 (CRYPTO_ENQ_FLUSH_THRESHOLD - 1)) == 0) {
 491                 nb_enqueued += eca_crypto_enq_flush(adapter);
 492         }
 493
 494         return nb_enqueued;
 495 }
 496
 497 static inline void
 498 eca_ops_enqueue_burst(struct rte_event_crypto_adapter *adapter,
 499                   struct rte_crypto_op **ops, uint16_t num)
 500 {
 501         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 502         union rte_event_crypto_metadata *m_data = NULL;
 503         uint8_t event_dev_id = adapter->eventdev_id;
 504         uint8_t event_port_id = adapter->event_port_id;
 505         struct rte_event events[BATCH_SIZE];
 506         uint16_t nb_enqueued, nb_ev;
 507         uint8_t retry;
 508         uint8_t i;
 509
 510         nb_ev = 0;
 511         retry = 0;
 512         nb_enqueued = 0;
 513         num = RTE_MIN(num, BATCH_SIZE);
 514         for (i = 0; i < num; i++) {
 515                 struct rte_event *ev = &events[nb_ev++];
 516                 if (ops[i]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 517                         m_data = rte_cryptodev_sym_session_get_user_data(
 518                                         ops[i]->sym->session);
 519                 } else if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS &&
 520                                 ops[i]->private_data_offset) {
 521                         m_data = (union rte_event_crypto_metadata *)
 522                                  ((uint8_t *)ops[i] +
 523                                   ops[i]->private_data_offset);
 524                 }
 525
 526                 if (unlikely(m_data == NULL)) {
 527                         rte_pktmbuf_free(ops[i]->sym->m_src);
 528                         rte_crypto_op_free(ops[i]);
 529                         continue;
 530                 }
 531
 532                 rte_memcpy(ev, &m_data->response_info, sizeof(*ev));
 533                 ev->event_ptr = ops[i];
 534                 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
 535                 if (adapter->implicit_release_disabled)
 536                         ev->op = RTE_EVENT_OP_FORWARD;
 537                 else
 538                         ev->op = RTE_EVENT_OP_NEW;
 539         }
 540
 541         do {
 542                 nb_enqueued += rte_event_enqueue_burst(event_dev_id,
 543                                                   event_port_id,
 544                                                   &events[nb_enqueued],
 545                                                   nb_ev - nb_enqueued);
 546         } while (retry++ < CRYPTO_ADAPTER_MAX_EV_ENQ_RETRIES &&
 547                  nb_enqueued < nb_ev);
 548
 549         /* Free mbufs and rte_crypto_ops for failed events */
 550         for (i = nb_enqueued; i < nb_ev; i++) {
 551                 struct rte_crypto_op *op = events[i].event_ptr;
 552                 rte_pktmbuf_free(op->sym->m_src);
 553                 rte_crypto_op_free(op);
 554         }
 555
 556         stats->event_enq_fail_count += nb_ev - nb_enqueued;
 557         stats->event_enq_count += nb_enqueued;
 558         stats->event_enq_retry_count += retry - 1;
 559 }
 560
 561 static inline unsigned int
 562 eca_crypto_adapter_deq_run(struct rte_event_crypto_adapter *adapter,
 563                         unsigned int max_deq)
 564 {
 565         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 566         struct crypto_device_info *curr_dev;
 567         struct crypto_queue_pair_info *curr_queue;
 568         struct rte_crypto_op *ops[BATCH_SIZE];
 569         uint16_t n, nb_deq;
 570         struct rte_cryptodev *dev;
 571         uint8_t cdev_id;
 572         uint16_t qp, dev_qps;
 573         bool done;
 574         uint16_t num_cdev = rte_cryptodev_count();
 575
 576         nb_deq = 0;
 577         do {
 578                 uint16_t queues = 0;
 579                 done = true;
 580
 581                 for (cdev_id = adapter->next_cdev_id;
 582                         cdev_id < num_cdev; cdev_id++) {
 583                         curr_dev = &adapter->cdevs[cdev_id];
 584                         dev = curr_dev->dev;
 585                         if (dev == NULL)
 586                                 continue;
 587                         dev_qps = dev->data->nb_queue_pairs;
 588
 589                         for (qp = curr_dev->next_queue_pair_id;
 590                                 queues < dev_qps; qp = (qp + 1) % dev_qps,
 591                                 queues++) {
 592
 593                                 curr_queue = &curr_dev->qpairs[qp];
 594                                 if (!curr_queue->qp_enabled)
 595                                         continue;
 596
 597                                 n = rte_cryptodev_dequeue_burst(cdev_id, qp,
 598                                         ops, BATCH_SIZE);
 599                                 if (!n)
 600                                         continue;
 601
 602                                 done = false;
 603                                 stats->crypto_deq_count += n;
 604                                 eca_ops_enqueue_burst(adapter, ops, n);
 605                                 nb_deq += n;
 606
 607                                 if (nb_deq > max_deq) {
 608                                         if ((qp + 1) == dev_qps) {
 609                                                 adapter->next_cdev_id =
 610                                                         (cdev_id + 1)
 611                                                         % num_cdev;
 612                                         }
 613                                         curr_dev->next_queue_pair_id = (qp + 1)
 614                                                 % dev->data->nb_queue_pairs;
 615
 616                                         return nb_deq;
 617                                 }
 618                         }
 619                 }
 620         } while (done == false);
 621         return nb_deq;
 622 }
 623
 624 static void
 625 eca_crypto_adapter_run(struct rte_event_crypto_adapter *adapter,
 626                         unsigned int max_ops)
 627 {
 628         while (max_ops) {
 629                 unsigned int e_cnt, d_cnt;
 630
 631                 e_cnt = eca_crypto_adapter_deq_run(adapter, max_ops);
 632                 max_ops -= RTE_MIN(max_ops, e_cnt);
 633
 634                 d_cnt = eca_crypto_adapter_enq_run(adapter, max_ops);
 635                 max_ops -= RTE_MIN(max_ops, d_cnt);
 636
 637                 if (e_cnt == 0 && d_cnt == 0)
 638                         break;
 639
 640         }
 641 }
 642
 643 static int
 644 eca_service_func(void *args)
 645 {
 646         struct rte_event_crypto_adapter *adapter = args;
 647
 648         if (rte_spinlock_trylock(&adapter->lock) == 0)
 649                 return 0;
 650         eca_crypto_adapter_run(adapter, adapter->max_nb);
 651         rte_spinlock_unlock(&adapter->lock);
 652
 653         return 0;
 654 }
 655
 656 static int
 657 eca_init_service(struct rte_event_crypto_adapter *adapter, uint8_t id)
 658 {
 659         struct rte_event_crypto_adapter_conf adapter_conf;
 660         struct rte_service_spec service;
 661         int ret;
 662
 663         if (adapter->service_inited)
 664                 return 0;
 665
 666         memset(&service, 0, sizeof(service));
 667         snprintf(service.name, CRYPTO_ADAPTER_NAME_LEN,
 668                 "rte_event_crypto_adapter_%d", id);
 669         service.socket_id = adapter->socket_id;
 670         service.callback = eca_service_func;
 671         service.callback_userdata = adapter;
 672         /* Service function handles locking for queue add/del updates */
 673         service.capabilities = RTE_SERVICE_CAP_MT_SAFE;
 674         ret = rte_service_component_register(&service, &adapter->service_id);
 675         if (ret) {
 676                 RTE_EDEV_LOG_ERR("failed to register service %s err = %" PRId32,
 677                         service.name, ret);
 678                 return ret;
 679         }
 680
 681         ret = adapter->conf_cb(id, adapter->eventdev_id,
 682                 &adapter_conf, adapter->conf_arg);
 683         if (ret) {
 684                 RTE_EDEV_LOG_ERR("configuration callback failed err = %" PRId32,
 685                         ret);
 686                 return ret;
 687         }
 688
 689         adapter->max_nb = adapter_conf.max_nb;
 690         adapter->event_port_id = adapter_conf.event_port_id;
 691         adapter->service_inited = 1;
 692
 693         return ret;
 694 }
 695
 696 static void
 697 eca_update_qp_info(struct rte_event_crypto_adapter *adapter,
 698                         struct crypto_device_info *dev_info,
 699                         int32_t queue_pair_id,
 700                         uint8_t add)
 701 {
 702         struct crypto_queue_pair_info *qp_info;
 703         int enabled;
 704         uint16_t i;
 705
 706         if (dev_info->qpairs == NULL)
 707                 return;
 708
 709         if (queue_pair_id == -1) {
 710                 for (i = 0; i < dev_info->dev->data->nb_queue_pairs; i++)
 711                         eca_update_qp_info(adapter, dev_info, i, add);
 712         } else {
 713                 qp_info = &dev_info->qpairs[queue_pair_id];
 714                 enabled = qp_info->qp_enabled;
 715                 if (add) {
 716                         adapter->nb_qps += !enabled;
 717                         dev_info->num_qpairs += !enabled;
 718                 } else {
 719                         adapter->nb_qps -= enabled;
 720                         dev_info->num_qpairs -= enabled;
 721                 }
 722                 qp_info->qp_enabled = !!add;
 723         }
 724 }
 725
 726 static int
 727 eca_add_queue_pair(struct rte_event_crypto_adapter *adapter,
 728                 uint8_t cdev_id,
 729                 int queue_pair_id)
 730 {
 731         struct crypto_device_info *dev_info = &adapter->cdevs[cdev_id];
 732         struct crypto_queue_pair_info *qpairs;
 733         uint32_t i;
 734
 735         if (dev_info->qpairs == NULL) {
 736                 dev_info->qpairs =
 737                     rte_zmalloc_socket(adapter->mem_name,
 738                                         dev_info->dev->data->nb_queue_pairs *
 739                                         sizeof(struct crypto_queue_pair_info),
 740                                         0, adapter->socket_id);
 741                 if (dev_info->qpairs == NULL)
 742                         return -ENOMEM;
 743
 744                 qpairs = dev_info->qpairs;
 745                 qpairs->op_buffer = rte_zmalloc_socket(adapter->mem_name,
 746                                         BATCH_SIZE *
 747                                         sizeof(struct rte_crypto_op *),
 748                                         0, adapter->socket_id);
 749                 if (!qpairs->op_buffer) {
 750                         rte_free(qpairs);
 751                         return -ENOMEM;
 752                 }
 753         }
 754
 755         if (queue_pair_id == -1) {
 756                 for (i = 0; i < dev_info->dev->data->nb_queue_pairs; i++)
 757                         eca_update_qp_info(adapter, dev_info, i, 1);
 758         } else
 759                 eca_update_qp_info(adapter, dev_info,
 760                                         (uint16_t)queue_pair_id, 1);
 761
 762         return 0;
 763 }
 764
 765 int __rte_experimental
 766 rte_event_crypto_adapter_queue_pair_add(uint8_t id,
 767                         uint8_t cdev_id,
 768                         int32_t queue_pair_id,
 769                         const struct rte_event *event)
 770 {
 771         struct rte_event_crypto_adapter *adapter;
 772         struct rte_eventdev *dev;
 773         struct crypto_device_info *dev_info;
 774         uint32_t cap;
 775         int ret;
 776
 777         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 778
 779         if (!rte_cryptodev_pmd_is_valid_dev(cdev_id)) {
 780                 RTE_EDEV_LOG_ERR("Invalid dev_id=%" PRIu8, cdev_id);
 781                 return -EINVAL;
 782         }
 783
 784         adapter = eca_id_to_adapter(id);
 785         if (adapter == NULL)
 786                 return -EINVAL;
 787
 788         dev = &rte_eventdevs[adapter->eventdev_id];
 789         ret = rte_event_crypto_adapter_caps_get(adapter->eventdev_id,
 790                                                 cdev_id,
 791                                                 &cap);
 792         if (ret) {
 793                 RTE_EDEV_LOG_ERR("Failed to get adapter caps dev %" PRIu8
 794                         " cdev %" PRIu8, id, cdev_id);
 795                 return ret;
 796         }
 797
 798         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) &&
 799             (event == NULL)) {
 800                 RTE_EDEV_LOG_ERR("Conf value can not be NULL for dev_id=%u",
 801                                   cdev_id);
 802                 return -EINVAL;
 803         }
 804
 805         dev_info = &adapter->cdevs[cdev_id];
 806
 807         if (queue_pair_id != -1 &&
 808             (uint16_t)queue_pair_id >= dev_info->dev->data->nb_queue_pairs) {
 809                 RTE_EDEV_LOG_ERR("Invalid queue_pair_id %" PRIu16,
 810                                  (uint16_t)queue_pair_id);
 811                 return -EINVAL;
 812         }
 813
 814         /* In case HW cap is RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD,
 815          * no need of service core as HW supports event forward capability.
 816          */
 817         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) ||
 818             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND &&
 819              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) ||
 820             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
 821              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)) {
 822                 RTE_FUNC_PTR_OR_ERR_RET(
 823                         *dev->dev_ops->crypto_adapter_queue_pair_add,
 824                         -ENOTSUP);
 825                 if (dev_info->qpairs == NULL) {
 826                         dev_info->qpairs =
 827                             rte_zmalloc_socket(adapter->mem_name,
 828                                         dev_info->dev->data->nb_queue_pairs *
 829                                         sizeof(struct crypto_queue_pair_info),
 830                                         0, adapter->socket_id);
 831                         if (dev_info->qpairs == NULL)
 832                                 return -ENOMEM;
 833                 }
 834
 835                 ret = (*dev->dev_ops->crypto_adapter_queue_pair_add)(dev,
 836                                 dev_info->dev,
 837                                 queue_pair_id,
 838                                 event);
 839                 if (ret)
 840                         return ret;
 841
 842                 else
 843                         eca_update_qp_info(adapter, &adapter->cdevs[cdev_id],
 844                                            queue_pair_id, 1);
 845         }
 846
 847         /* In case HW cap is RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW,
 848          * or SW adapter, initiate services so the application can choose
 849          * which ever way it wants to use the adapter.
 850          * Case 1: RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW
 851          *         Application may wants to use one of below two mode
 852          *          a. OP_FORWARD mode -> HW Dequeue + SW enqueue
 853          *          b. OP_NEW mode -> HW Dequeue
 854          * Case 2: No HW caps, use SW adapter
 855          *          a. OP_FORWARD mode -> SW enqueue & dequeue
 856          *          b. OP_NEW mode -> SW Dequeue
 857          */
 858         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
 859              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_FORWARD) ||
 860              (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW) &&
 861               !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) &&
 862               !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) &&
 863                (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA))) {
 864                 rte_spinlock_lock(&adapter->lock);
 865                 ret = eca_init_service(adapter, id);
 866                 if (ret == 0)
 867                         ret = eca_add_queue_pair(adapter, cdev_id,
 868                                                  queue_pair_id);
 869                 rte_spinlock_unlock(&adapter->lock);
 870
 871                 if (ret)
 872                         return ret;
 873
 874                 rte_service_component_runstate_set(adapter->service_id, 1);
 875         }
 876
 877         return 0;
 878 }
 879
 880 int __rte_experimental
 881 rte_event_crypto_adapter_queue_pair_del(uint8_t id, uint8_t cdev_id,
 882                                         int32_t queue_pair_id)
 883 {
 884         struct rte_event_crypto_adapter *adapter;
 885         struct crypto_device_info *dev_info;
 886         struct rte_eventdev *dev;
 887         int ret;
 888         uint32_t cap;
 889         uint16_t i;
 890
 891         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 892
 893         if (!rte_cryptodev_pmd_is_valid_dev(cdev_id)) {
 894                 RTE_EDEV_LOG_ERR("Invalid dev_id=%" PRIu8, cdev_id);
 895                 return -EINVAL;
 896         }
 897
 898         adapter = eca_id_to_adapter(id);
 899         if (adapter == NULL)
 900                 return -EINVAL;
 901
 902         dev = &rte_eventdevs[adapter->eventdev_id];
 903         ret = rte_event_crypto_adapter_caps_get(adapter->eventdev_id,
 904                                                 cdev_id,
 905                                                 &cap);
 906         if (ret)
 907                 return ret;
 908
 909         dev_info = &adapter->cdevs[cdev_id];
 910
 911         if (queue_pair_id != -1 &&
 912             (uint16_t)queue_pair_id >= dev_info->dev->data->nb_queue_pairs) {
 913                 RTE_EDEV_LOG_ERR("Invalid queue_pair_id %" PRIu16,
 914                                  (uint16_t)queue_pair_id);
 915                 return -EINVAL;
 916         }
 917
 918         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) ||
 919             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
 920              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)) {
 921                 RTE_FUNC_PTR_OR_ERR_RET(
 922                         *dev->dev_ops->crypto_adapter_queue_pair_del,
 923                         -ENOTSUP);
 924                 ret = (*dev->dev_ops->crypto_adapter_queue_pair_del)(dev,
 925                                                 dev_info->dev,
 926                                                 queue_pair_id);
 927                 if (ret == 0) {
 928                         eca_update_qp_info(adapter,
 929                                         &adapter->cdevs[cdev_id],
 930                                         queue_pair_id,
 931                                         0);
 932                         if (dev_info->num_qpairs == 0) {
 933                                 rte_free(dev_info->qpairs);
 934                                 dev_info->qpairs = NULL;
 935                         }
 936                 }
 937         } else {
 938                 if (adapter->nb_qps == 0)
 939                         return 0;
 940
 941                 rte_spinlock_lock(&adapter->lock);
 942                 if (queue_pair_id == -1) {
 943                         for (i = 0; i < dev_info->dev->data->nb_queue_pairs;
 944                                 i++)
 945                                 eca_update_qp_info(adapter, dev_info,
 946                                                         queue_pair_id, 0);
 947                 } else {
 948                         eca_update_qp_info(adapter, dev_info,
 949                                                 (uint16_t)queue_pair_id, 0);
 950                 }
 951
 952                 if (dev_info->num_qpairs == 0) {
 953                         rte_free(dev_info->qpairs);
 954                         dev_info->qpairs = NULL;
 955                 }
 956
 957                 rte_spinlock_unlock(&adapter->lock);
 958                 rte_service_component_runstate_set(adapter->service_id,
 959                                 adapter->nb_qps);
 960         }
 961
 962         return ret;
 963 }
 964
 965 static int
 966 eca_adapter_ctrl(uint8_t id, int start)
 967 {
 968         struct rte_event_crypto_adapter *adapter;
 969         struct crypto_device_info *dev_info;
 970         struct rte_eventdev *dev;
 971         uint32_t i;
 972         int use_service;
 973         int stop = !start;
 974
 975         use_service = 0;
 976         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 977         adapter = eca_id_to_adapter(id);
 978         if (adapter == NULL)
 979                 return -EINVAL;
 980
 981         dev = &rte_eventdevs[adapter->eventdev_id];
 982
 983         for (i = 0; i < rte_cryptodev_count(); i++) {
 984                 dev_info = &adapter->cdevs[i];
 985                 /* if start  check for num queue pairs */
 986                 if (start && !dev_info->num_qpairs)
 987                         continue;
 988                 /* if stop check if dev has been started */
 989                 if (stop && !dev_info->dev_started)
 990                         continue;
 991                 use_service |= !dev_info->internal_event_port;
 992                 dev_info->dev_started = start;
 993                 if (dev_info->internal_event_port == 0)
 994                         continue;
 995                 start ? (*dev->dev_ops->crypto_adapter_start)(dev,
 996                                                 &dev_info->dev[i]) :
 997                         (*dev->dev_ops->crypto_adapter_stop)(dev,
 998                                                 &dev_info->dev[i]);
 999         }
1000
1001         if (use_service)
1002                 rte_service_runstate_set(adapter->service_id, start);
1003
1004         return 0;
1005 }
1006
1007 int __rte_experimental
1008 rte_event_crypto_adapter_start(uint8_t id)
1009 {
1010         struct rte_event_crypto_adapter *adapter;
1011
1012         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1013         adapter = eca_id_to_adapter(id);
1014         if (adapter == NULL)
1015                 return -EINVAL;
1016
1017         return eca_adapter_ctrl(id, 1);
1018 }
1019
1020 int __rte_experimental
1021 rte_event_crypto_adapter_stop(uint8_t id)
1022 {
1023         return eca_adapter_ctrl(id, 0);
1024 }
1025
1026 int __rte_experimental
1027 rte_event_crypto_adapter_stats_get(uint8_t id,
1028                                 struct rte_event_crypto_adapter_stats *stats)
1029 {
1030         struct rte_event_crypto_adapter *adapter;
1031         struct rte_event_crypto_adapter_stats dev_stats_sum = { 0 };
1032         struct rte_event_crypto_adapter_stats dev_stats;
1033         struct rte_eventdev *dev;
1034         struct crypto_device_info *dev_info;
1035         uint32_t i;
1036         int ret;
1037
1038         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1039
1040         adapter = eca_id_to_adapter(id);
1041         if (adapter == NULL || stats == NULL)
1042                 return -EINVAL;
1043
1044         dev = &rte_eventdevs[adapter->eventdev_id];
1045         memset(stats, 0, sizeof(*stats));
1046         for (i = 0; i < rte_cryptodev_count(); i++) {
1047                 dev_info = &adapter->cdevs[i];
1048                 if (dev_info->internal_event_port == 0 ||
1049                         dev->dev_ops->crypto_adapter_stats_get == NULL)
1050                         continue;
1051                 ret = (*dev->dev_ops->crypto_adapter_stats_get)(dev,
1052                                                 dev_info->dev,
1053                                                 &dev_stats);
1054                 if (ret)
1055                         continue;
1056
1057                 dev_stats_sum.crypto_deq_count += dev_stats.crypto_deq_count;
1058                 dev_stats_sum.event_enq_count +=
1059                         dev_stats.event_enq_count;
1060         }
1061
1062         if (adapter->service_inited)
1063                 *stats = adapter->crypto_stats;
1064
1065         stats->crypto_deq_count += dev_stats_sum.crypto_deq_count;
1066         stats->event_enq_count += dev_stats_sum.event_enq_count;
1067
1068         return 0;
1069 }
1070
1071 int __rte_experimental
1072 rte_event_crypto_adapter_stats_reset(uint8_t id)
1073 {
1074         struct rte_event_crypto_adapter *adapter;
1075         struct crypto_device_info *dev_info;
1076         struct rte_eventdev *dev;
1077         uint32_t i;
1078
1079         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1080
1081         adapter = eca_id_to_adapter(id);
1082         if (adapter == NULL)
1083                 return -EINVAL;
1084
1085         dev = &rte_eventdevs[adapter->eventdev_id];
1086         for (i = 0; i < rte_cryptodev_count(); i++) {
1087                 dev_info = &adapter->cdevs[i];
1088                 if (dev_info->internal_event_port == 0 ||
1089                         dev->dev_ops->crypto_adapter_stats_reset == NULL)
1090                         continue;
1091                 (*dev->dev_ops->crypto_adapter_stats_reset)(dev,
1092                                                 dev_info->dev);
1093         }
1094
1095         memset(&adapter->crypto_stats, 0, sizeof(adapter->crypto_stats));
1096         return 0;
1097 }
1098
1099 int __rte_experimental
1100 rte_event_crypto_adapter_service_id_get(uint8_t id, uint32_t *service_id)
1101 {
1102         struct rte_event_crypto_adapter *adapter;
1103
1104         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1105
1106         adapter = eca_id_to_adapter(id);
1107         if (adapter == NULL || service_id == NULL)
1108                 return -EINVAL;
1109
1110         if (adapter->service_inited)
1111                 *service_id = adapter->service_id;
1112
1113         return adapter->service_inited ? 0 : -ESRCH;
1114 }
1115
1116 int __rte_experimental
1117 rte_event_crypto_adapter_event_port_get(uint8_t id, uint8_t *event_port_id)
1118 {
1119         struct rte_event_crypto_adapter *adapter;
1120
1121         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1122
1123         adapter = eca_id_to_adapter(id);
1124         if (adapter == NULL || event_port_id == NULL)
1125                 return -EINVAL;
1126
1127         *event_port_id = adapter->event_port_id;
1128
1129         return 0;
1130 }