lib/librte_eventdev/rte_event_crypto_adapter.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation.
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <stdbool.h>
   8 #include <rte_common.h>
   9 #include <rte_dev.h>
  10 #include <rte_errno.h>
  11 #include <rte_cryptodev.h>
  12 #include <rte_cryptodev_pmd.h>
  13 #include <rte_log.h>
  14 #include <rte_malloc.h>
  15 #include <rte_service_component.h>
  16
  17 #include "rte_eventdev.h"
  18 #include "rte_eventdev_pmd.h"
  19 #include "rte_event_crypto_adapter.h"
  20
  21 #define BATCH_SIZE 32
  22 #define DEFAULT_MAX_NB 128
  23 #define CRYPTO_ADAPTER_NAME_LEN 32
  24 #define CRYPTO_ADAPTER_MEM_NAME_LEN 32
  25 #define CRYPTO_ADAPTER_MAX_EV_ENQ_RETRIES 100
  26
  27 /* Flush an instance's enqueue buffers every CRYPTO_ENQ_FLUSH_THRESHOLD
  28  * iterations of eca_crypto_adapter_enq_run()
  29  */
  30 #define CRYPTO_ENQ_FLUSH_THRESHOLD 1024
  31
  32 struct rte_event_crypto_adapter {
  33         /* Event device identifier */
  34         uint8_t eventdev_id;
  35         /* Event port identifier */
  36         uint8_t event_port_id;
  37         /* Store event device's implicit release capability */
  38         uint8_t implicit_release_disabled;
  39         /* Max crypto ops processed in any service function invocation */
  40         uint32_t max_nb;
  41         /* Lock to serialize config updates with service function */
  42         rte_spinlock_t lock;
  43         /* Next crypto device to be processed */
  44         uint16_t next_cdev_id;
  45         /* Per crypto device structure */
  46         struct crypto_device_info *cdevs;
  47         /* Loop counter to flush crypto ops */
  48         uint16_t transmit_loop_count;
  49         /* Per instance stats structure */
  50         struct rte_event_crypto_adapter_stats crypto_stats;
  51         /* Configuration callback for rte_service configuration */
  52         rte_event_crypto_adapter_conf_cb conf_cb;
  53         /* Configuration callback argument */
  54         void *conf_arg;
  55         /* Set if  default_cb is being used */
  56         int default_cb_arg;
  57         /* Service initialization state */
  58         uint8_t service_inited;
  59         /* Memory allocation name */
  60         char mem_name[CRYPTO_ADAPTER_MEM_NAME_LEN];
  61         /* Socket identifier cached from eventdev */
  62         int socket_id;
  63         /* Per adapter EAL service */
  64         uint32_t service_id;
  65         /* No. of queue pairs configured */
  66         uint16_t nb_qps;
  67         /* Adapter mode */
  68         enum rte_event_crypto_adapter_mode mode;
  69 } __rte_cache_aligned;
  70
  71 /* Per crypto device information */
  72 struct crypto_device_info {
  73         /* Pointer to cryptodev */
  74         struct rte_cryptodev *dev;
  75         /* Pointer to queue pair info */
  76         struct crypto_queue_pair_info *qpairs;
  77         /* Next queue pair to be processed */
  78         uint16_t next_queue_pair_id;
  79         /* Set to indicate cryptodev->eventdev packet
  80          * transfer uses a hardware mechanism
  81          */
  82         uint8_t internal_event_port;
  83         /* Set to indicate processing has been started */
  84         uint8_t dev_started;
  85         /* If num_qpairs > 0, the start callback will
  86          * be invoked if not already invoked
  87          */
  88         uint16_t num_qpairs;
  89 } __rte_cache_aligned;
  90
  91 /* Per queue pair information */
  92 struct crypto_queue_pair_info {
  93         /* Set to indicate queue pair is enabled */
  94         bool qp_enabled;
  95         /* Pointer to hold rte_crypto_ops for batching */
  96         struct rte_crypto_op **op_buffer;
  97         /* No of crypto ops accumulated */
  98         uint8_t len;
  99 } __rte_cache_aligned;
 100
 101 static struct rte_event_crypto_adapter **event_crypto_adapter;
 102
 103 /* Macros to check for valid adapter */
 104 #define EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \
 105         if (!eca_valid_id(id)) { \
 106                 RTE_EDEV_LOG_ERR("Invalid crypto adapter id = %d\n", id); \
 107                 return retval; \
 108         } \
 109 } while (0)
 110
 111 static inline int
 112 eca_valid_id(uint8_t id)
 113 {
 114         return id < RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE;
 115 }
 116
 117 static int
 118 eca_init(void)
 119 {
 120         const char *name = "crypto_adapter_array";
 121         const struct rte_memzone *mz;
 122         unsigned int sz;
 123
 124         sz = sizeof(*event_crypto_adapter) *
 125             RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE;
 126         sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE);
 127
 128         mz = rte_memzone_lookup(name);
 129         if (mz == NULL) {
 130                 mz = rte_memzone_reserve_aligned(name, sz, rte_socket_id(), 0,
 131                                                  RTE_CACHE_LINE_SIZE);
 132                 if (mz == NULL) {
 133                         RTE_EDEV_LOG_ERR("failed to reserve memzone err = %"
 134                                         PRId32, rte_errno);
 135                         return -rte_errno;
 136                 }
 137         }
 138
 139         event_crypto_adapter = mz->addr;
 140         return 0;
 141 }
 142
 143 static inline struct rte_event_crypto_adapter *
 144 eca_id_to_adapter(uint8_t id)
 145 {
 146         return event_crypto_adapter ?
 147                 event_crypto_adapter[id] : NULL;
 148 }
 149
 150 static int
 151 eca_default_config_cb(uint8_t id, uint8_t dev_id,
 152                         struct rte_event_crypto_adapter_conf *conf, void *arg)
 153 {
 154         struct rte_event_dev_config dev_conf;
 155         struct rte_eventdev *dev;
 156         uint8_t port_id;
 157         int started;
 158         int ret;
 159         struct rte_event_port_conf *port_conf = arg;
 160         struct rte_event_crypto_adapter *adapter = eca_id_to_adapter(id);
 161
 162         dev = &rte_eventdevs[adapter->eventdev_id];
 163         dev_conf = dev->data->dev_conf;
 164
 165         started = dev->data->dev_started;
 166         if (started)
 167                 rte_event_dev_stop(dev_id);
 168         port_id = dev_conf.nb_event_ports;
 169         dev_conf.nb_event_ports += 1;
 170         ret = rte_event_dev_configure(dev_id, &dev_conf);
 171         if (ret) {
 172                 RTE_EDEV_LOG_ERR("failed to configure event dev %u\n", dev_id);
 173                 if (started) {
 174                         if (rte_event_dev_start(dev_id))
 175                                 return -EIO;
 176                 }
 177                 return ret;
 178         }
 179
 180         ret = rte_event_port_setup(dev_id, port_id, port_conf);
 181         if (ret) {
 182                 RTE_EDEV_LOG_ERR("failed to setup event port %u\n", port_id);
 183                 return ret;
 184         }
 185
 186         conf->event_port_id = port_id;
 187         conf->max_nb = DEFAULT_MAX_NB;
 188         if (started)
 189                 ret = rte_event_dev_start(dev_id);
 190
 191         adapter->default_cb_arg = 1;
 192         return ret;
 193 }
 194
 195 int __rte_experimental
 196 rte_event_crypto_adapter_create_ext(uint8_t id, uint8_t dev_id,
 197                                 rte_event_crypto_adapter_conf_cb conf_cb,
 198                                 enum rte_event_crypto_adapter_mode mode,
 199                                 void *conf_arg)
 200 {
 201         struct rte_event_crypto_adapter *adapter;
 202         char mem_name[CRYPTO_ADAPTER_NAME_LEN];
 203         struct rte_event_dev_info dev_info;
 204         int socket_id;
 205         uint8_t i;
 206         int ret;
 207
 208         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 209         RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
 210         if (conf_cb == NULL)
 211                 return -EINVAL;
 212
 213         if (event_crypto_adapter == NULL) {
 214                 ret = eca_init();
 215                 if (ret)
 216                         return ret;
 217         }
 218
 219         adapter = eca_id_to_adapter(id);
 220         if (adapter != NULL) {
 221                 RTE_EDEV_LOG_ERR("Crypto adapter id %u already exists!", id);
 222                 return -EEXIST;
 223         }
 224
 225         socket_id = rte_event_dev_socket_id(dev_id);
 226         snprintf(mem_name, CRYPTO_ADAPTER_MEM_NAME_LEN,
 227                  "rte_event_crypto_adapter_%d", id);
 228
 229         adapter = rte_zmalloc_socket(mem_name, sizeof(*adapter),
 230                         RTE_CACHE_LINE_SIZE, socket_id);
 231         if (adapter == NULL) {
 232                 RTE_EDEV_LOG_ERR("Failed to get mem for event crypto adapter!");
 233                 return -ENOMEM;
 234         }
 235
 236         ret = rte_event_dev_info_get(dev_id, &dev_info);
 237         if (ret < 0) {
 238                 RTE_EDEV_LOG_ERR("Failed to get info for eventdev %d: %s!",
 239                                  dev_id, dev_info.driver_name);
 240                 return ret;
 241         }
 242
 243         adapter->implicit_release_disabled = (dev_info.event_dev_cap &
 244                         RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE);
 245         adapter->eventdev_id = dev_id;
 246         adapter->socket_id = socket_id;
 247         adapter->conf_cb = conf_cb;
 248         adapter->conf_arg = conf_arg;
 249         adapter->mode = mode;
 250         strcpy(adapter->mem_name, mem_name);
 251         adapter->cdevs = rte_zmalloc_socket(adapter->mem_name,
 252                                         rte_cryptodev_count() *
 253                                         sizeof(struct crypto_device_info), 0,
 254                                         socket_id);
 255         if (adapter->cdevs == NULL) {
 256                 RTE_EDEV_LOG_ERR("Failed to get mem for crypto devices\n");
 257                 rte_free(adapter);
 258                 return -ENOMEM;
 259         }
 260
 261         rte_spinlock_init(&adapter->lock);
 262         for (i = 0; i < rte_cryptodev_count(); i++)
 263                 adapter->cdevs[i].dev = rte_cryptodev_pmd_get_dev(i);
 264
 265         event_crypto_adapter[id] = adapter;
 266
 267         return 0;
 268 }
 269
 270
 271 int __rte_experimental
 272 rte_event_crypto_adapter_create(uint8_t id, uint8_t dev_id,
 273                                 struct rte_event_port_conf *port_config,
 274                                 enum rte_event_crypto_adapter_mode mode)
 275 {
 276         struct rte_event_port_conf *pc;
 277         int ret;
 278
 279         if (port_config == NULL)
 280                 return -EINVAL;
 281         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 282
 283         pc = rte_malloc(NULL, sizeof(*pc), 0);
 284         if (pc == NULL)
 285                 return -ENOMEM;
 286         *pc = *port_config;
 287         ret = rte_event_crypto_adapter_create_ext(id, dev_id,
 288                                                   eca_default_config_cb,
 289                                                   mode,
 290                                                   pc);
 291         if (ret)
 292                 rte_free(pc);
 293
 294         return ret;
 295 }
 296
 297 int __rte_experimental
 298 rte_event_crypto_adapter_free(uint8_t id)
 299 {
 300         struct rte_event_crypto_adapter *adapter;
 301
 302         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 303
 304         adapter = eca_id_to_adapter(id);
 305         if (adapter == NULL)
 306                 return -EINVAL;
 307
 308         if (adapter->nb_qps) {
 309                 RTE_EDEV_LOG_ERR("%" PRIu16 "Queue pairs not deleted",
 310                                 adapter->nb_qps);
 311                 return -EBUSY;
 312         }
 313
 314         if (adapter->default_cb_arg)
 315                 rte_free(adapter->conf_arg);
 316         rte_free(adapter->cdevs);
 317         rte_free(adapter);
 318         event_crypto_adapter[id] = NULL;
 319
 320         return 0;
 321 }
 322
 323 static inline unsigned int
 324 eca_enq_to_cryptodev(struct rte_event_crypto_adapter *adapter,
 325                  struct rte_event *ev, unsigned int cnt)
 326 {
 327         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 328         union rte_event_crypto_metadata *m_data = NULL;
 329         struct crypto_queue_pair_info *qp_info = NULL;
 330         struct rte_crypto_op *crypto_op;
 331         unsigned int i, n;
 332         uint16_t qp_id, len, ret;
 333         uint8_t cdev_id;
 334
 335         len = 0;
 336         ret = 0;
 337         n = 0;
 338         stats->event_deq_count += cnt;
 339
 340         for (i = 0; i < cnt; i++) {
 341                 crypto_op = ev[i].event_ptr;
 342                 if (crypto_op == NULL)
 343                         continue;
 344                 if (crypto_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 345                         m_data = rte_cryptodev_sym_session_get_user_data(
 346                                         crypto_op->sym->session);
 347                         if (m_data == NULL) {
 348                                 rte_pktmbuf_free(crypto_op->sym->m_src);
 349                                 rte_crypto_op_free(crypto_op);
 350                                 continue;
 351                         }
 352
 353                         cdev_id = m_data->request_info.cdev_id;
 354                         qp_id = m_data->request_info.queue_pair_id;
 355                         qp_info = &adapter->cdevs[cdev_id].qpairs[qp_id];
 356                         if (qp_info == NULL) {
 357                                 rte_pktmbuf_free(crypto_op->sym->m_src);
 358                                 rte_crypto_op_free(crypto_op);
 359                                 continue;
 360                         }
 361                         len = qp_info->len;
 362                         qp_info->op_buffer[len] = crypto_op;
 363                         len++;
 364                 } else if (crypto_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS &&
 365                                 crypto_op->private_data_offset) {
 366                         m_data = (union rte_event_crypto_metadata *)
 367                                  ((uint8_t *)crypto_op +
 368                                         crypto_op->private_data_offset);
 369                         cdev_id = m_data->request_info.cdev_id;
 370                         qp_id = m_data->request_info.queue_pair_id;
 371                         qp_info = &adapter->cdevs[cdev_id].qpairs[qp_id];
 372                         if (qp_info == NULL) {
 373                                 rte_pktmbuf_free(crypto_op->sym->m_src);
 374                                 rte_crypto_op_free(crypto_op);
 375                                 continue;
 376                         }
 377                         len = qp_info->len;
 378                         qp_info->op_buffer[len] = crypto_op;
 379                         len++;
 380                 } else {
 381                         rte_pktmbuf_free(crypto_op->sym->m_src);
 382                         rte_crypto_op_free(crypto_op);
 383                         continue;
 384                 }
 385
 386                 if (len == BATCH_SIZE) {
 387                         struct rte_crypto_op **op_buffer = qp_info->op_buffer;
 388                         ret = rte_cryptodev_enqueue_burst(cdev_id,
 389                                                           qp_id,
 390                                                           op_buffer,
 391                                                           BATCH_SIZE);
 392
 393                         stats->crypto_enq_count += ret;
 394
 395                         while (ret < len) {
 396                                 struct rte_crypto_op *op;
 397                                 op = op_buffer[ret++];
 398                                 stats->crypto_enq_fail++;
 399                                 rte_pktmbuf_free(op->sym->m_src);
 400                                 rte_crypto_op_free(op);
 401                         }
 402
 403                         len = 0;
 404                 }
 405
 406                 if (qp_info)
 407                         qp_info->len = len;
 408                 n += ret;
 409         }
 410
 411         return n;
 412 }
 413
 414 static unsigned int
 415 eca_crypto_enq_flush(struct rte_event_crypto_adapter *adapter)
 416 {
 417         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 418         struct crypto_device_info *curr_dev;
 419         struct crypto_queue_pair_info *curr_queue;
 420         struct rte_crypto_op **op_buffer;
 421         struct rte_cryptodev *dev;
 422         uint8_t cdev_id;
 423         uint16_t qp;
 424         uint16_t ret;
 425         uint16_t num_cdev = rte_cryptodev_count();
 426
 427         ret = 0;
 428         for (cdev_id = 0; cdev_id < num_cdev; cdev_id++) {
 429                 curr_dev = &adapter->cdevs[cdev_id];
 430                 if (curr_dev == NULL)
 431                         continue;
 432                 dev = curr_dev->dev;
 433
 434                 for (qp = 0; qp < dev->data->nb_queue_pairs; qp++) {
 435
 436                         curr_queue = &curr_dev->qpairs[qp];
 437                         if (!curr_queue->qp_enabled)
 438                                 continue;
 439
 440                         op_buffer = curr_queue->op_buffer;
 441                         ret = rte_cryptodev_enqueue_burst(cdev_id,
 442                                                           qp,
 443                                                           op_buffer,
 444                                                           curr_queue->len);
 445                         stats->crypto_enq_count += ret;
 446
 447                         while (ret < curr_queue->len) {
 448                                 struct rte_crypto_op *op;
 449                                 op = op_buffer[ret++];
 450                                 stats->crypto_enq_fail++;
 451                                 rte_pktmbuf_free(op->sym->m_src);
 452                                 rte_crypto_op_free(op);
 453                         }
 454                         curr_queue->len = 0;
 455                 }
 456         }
 457
 458         return ret;
 459 }
 460
 461 static int
 462 eca_crypto_adapter_enq_run(struct rte_event_crypto_adapter *adapter,
 463                         unsigned int max_enq)
 464 {
 465         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 466         struct rte_event ev[BATCH_SIZE];
 467         unsigned int nb_enq, nb_enqueued;
 468         uint16_t n;
 469         uint8_t event_dev_id = adapter->eventdev_id;
 470         uint8_t event_port_id = adapter->event_port_id;
 471
 472         nb_enqueued = 0;
 473         if (adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)
 474                 return 0;
 475
 476         for (nb_enq = 0; nb_enq < max_enq; nb_enq += n) {
 477                 stats->event_poll_count++;
 478                 n = rte_event_dequeue_burst(event_dev_id,
 479                                             event_port_id, ev, BATCH_SIZE, 0);
 480
 481                 if (!n)
 482                         break;
 483
 484                 nb_enqueued += eca_enq_to_cryptodev(adapter, ev, n);
 485         }
 486
 487         if ((++adapter->transmit_loop_count &
 488                 (CRYPTO_ENQ_FLUSH_THRESHOLD - 1)) == 0) {
 489                 nb_enqueued += eca_crypto_enq_flush(adapter);
 490         }
 491
 492         return nb_enqueued;
 493 }
 494
 495 static inline void
 496 eca_ops_enqueue_burst(struct rte_event_crypto_adapter *adapter,
 497                   struct rte_crypto_op **ops, uint16_t num)
 498 {
 499         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 500         union rte_event_crypto_metadata *m_data = NULL;
 501         uint8_t event_dev_id = adapter->eventdev_id;
 502         uint8_t event_port_id = adapter->event_port_id;
 503         struct rte_event events[BATCH_SIZE];
 504         uint16_t nb_enqueued, nb_ev;
 505         uint8_t retry;
 506         uint8_t i;
 507
 508         nb_ev = 0;
 509         retry = 0;
 510         nb_enqueued = 0;
 511         num = RTE_MIN(num, BATCH_SIZE);
 512         for (i = 0; i < num; i++) {
 513                 struct rte_event *ev = &events[nb_ev++];
 514                 if (ops[i]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 515                         m_data = rte_cryptodev_sym_session_get_user_data(
 516                                         ops[i]->sym->session);
 517                 } else if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS &&
 518                                 ops[i]->private_data_offset) {
 519                         m_data = (union rte_event_crypto_metadata *)
 520                                  ((uint8_t *)ops[i] +
 521                                   ops[i]->private_data_offset);
 522                 }
 523
 524                 if (unlikely(m_data == NULL)) {
 525                         rte_pktmbuf_free(ops[i]->sym->m_src);
 526                         rte_crypto_op_free(ops[i]);
 527                         continue;
 528                 }
 529
 530                 rte_memcpy(ev, &m_data->response_info, sizeof(*ev));
 531                 ev->event_ptr = ops[i];
 532                 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
 533                 if (adapter->implicit_release_disabled)
 534                         ev->op = RTE_EVENT_OP_FORWARD;
 535                 else
 536                         ev->op = RTE_EVENT_OP_NEW;
 537         }
 538
 539         do {
 540                 nb_enqueued += rte_event_enqueue_burst(event_dev_id,
 541                                                   event_port_id,
 542                                                   &events[nb_enqueued],
 543                                                   nb_ev - nb_enqueued);
 544         } while (retry++ < CRYPTO_ADAPTER_MAX_EV_ENQ_RETRIES &&
 545                  nb_enqueued < nb_ev);
 546
 547         /* Free mbufs and rte_crypto_ops for failed events */
 548         for (i = nb_enqueued; i < nb_ev; i++) {
 549                 struct rte_crypto_op *op = events[i].event_ptr;
 550                 rte_pktmbuf_free(op->sym->m_src);
 551                 rte_crypto_op_free(op);
 552         }
 553
 554         stats->event_enq_fail_count += nb_ev - nb_enqueued;
 555         stats->event_enq_count += nb_enqueued;
 556         stats->event_enq_retry_count += retry - 1;
 557 }
 558
 559 static inline unsigned int
 560 eca_crypto_adapter_deq_run(struct rte_event_crypto_adapter *adapter,
 561                         unsigned int max_deq)
 562 {
 563         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 564         struct crypto_device_info *curr_dev;
 565         struct crypto_queue_pair_info *curr_queue;
 566         struct rte_crypto_op *ops[BATCH_SIZE];
 567         uint16_t n, nb_deq;
 568         struct rte_cryptodev *dev;
 569         uint8_t cdev_id;
 570         uint16_t qp, dev_qps;
 571         bool done;
 572         uint16_t num_cdev = rte_cryptodev_count();
 573
 574         nb_deq = 0;
 575         do {
 576                 uint16_t queues = 0;
 577                 done = true;
 578
 579                 for (cdev_id = adapter->next_cdev_id;
 580                         cdev_id < num_cdev; cdev_id++) {
 581                         curr_dev = &adapter->cdevs[cdev_id];
 582                         if (curr_dev == NULL)
 583                                 continue;
 584                         dev = curr_dev->dev;
 585                         dev_qps = dev->data->nb_queue_pairs;
 586
 587                         for (qp = curr_dev->next_queue_pair_id;
 588                                 queues < dev_qps; qp = (qp + 1) % dev_qps,
 589                                 queues++) {
 590
 591                                 curr_queue = &curr_dev->qpairs[qp];
 592                                 if (!curr_queue->qp_enabled)
 593                                         continue;
 594
 595                                 n = rte_cryptodev_dequeue_burst(cdev_id, qp,
 596                                         ops, BATCH_SIZE);
 597                                 if (!n)
 598                                         continue;
 599
 600                                 done = false;
 601                                 stats->crypto_deq_count += n;
 602                                 eca_ops_enqueue_burst(adapter, ops, n);
 603                                 nb_deq += n;
 604
 605                                 if (nb_deq > max_deq) {
 606                                         if ((qp + 1) == dev_qps) {
 607                                                 adapter->next_cdev_id =
 608                                                         (cdev_id + 1)
 609                                                         % num_cdev;
 610                                         }
 611                                         curr_dev->next_queue_pair_id = (qp + 1)
 612                                                 % dev->data->nb_queue_pairs;
 613
 614                                         return nb_deq;
 615                                 }
 616                         }
 617                 }
 618         } while (done == false);
 619         return nb_deq;
 620 }
 621
 622 static void
 623 eca_crypto_adapter_run(struct rte_event_crypto_adapter *adapter,
 624                         unsigned int max_ops)
 625 {
 626         while (max_ops) {
 627                 unsigned int e_cnt, d_cnt;
 628
 629                 e_cnt = eca_crypto_adapter_deq_run(adapter, max_ops);
 630                 max_ops -= RTE_MIN(max_ops, e_cnt);
 631
 632                 d_cnt = eca_crypto_adapter_enq_run(adapter, max_ops);
 633                 max_ops -= RTE_MIN(max_ops, d_cnt);
 634
 635                 if (e_cnt == 0 && d_cnt == 0)
 636                         break;
 637
 638         }
 639 }
 640
 641 static int
 642 eca_service_func(void *args)
 643 {
 644         struct rte_event_crypto_adapter *adapter = args;
 645
 646         if (rte_spinlock_trylock(&adapter->lock) == 0)
 647                 return 0;
 648         eca_crypto_adapter_run(adapter, adapter->max_nb);
 649         rte_spinlock_unlock(&adapter->lock);
 650
 651         return 0;
 652 }
 653
 654 static int
 655 eca_init_service(struct rte_event_crypto_adapter *adapter, uint8_t id)
 656 {
 657         struct rte_event_crypto_adapter_conf adapter_conf;
 658         struct rte_service_spec service;
 659         int ret;
 660
 661         if (adapter->service_inited)
 662                 return 0;
 663
 664         memset(&service, 0, sizeof(service));
 665         snprintf(service.name, CRYPTO_ADAPTER_NAME_LEN,
 666                 "rte_event_crypto_adapter_%d", id);
 667         service.socket_id = adapter->socket_id;
 668         service.callback = eca_service_func;
 669         service.callback_userdata = adapter;
 670         /* Service function handles locking for queue add/del updates */
 671         service.capabilities = RTE_SERVICE_CAP_MT_SAFE;
 672         ret = rte_service_component_register(&service, &adapter->service_id);
 673         if (ret) {
 674                 RTE_EDEV_LOG_ERR("failed to register service %s err = %" PRId32,
 675                         service.name, ret);
 676                 return ret;
 677         }
 678
 679         ret = adapter->conf_cb(id, adapter->eventdev_id,
 680                 &adapter_conf, adapter->conf_arg);
 681         if (ret) {
 682                 RTE_EDEV_LOG_ERR("configuration callback failed err = %" PRId32,
 683                         ret);
 684                 return ret;
 685         }
 686
 687         adapter->max_nb = adapter_conf.max_nb;
 688         adapter->event_port_id = adapter_conf.event_port_id;
 689         adapter->service_inited = 1;
 690
 691         return ret;
 692 }
 693
 694 static void
 695 eca_update_qp_info(struct rte_event_crypto_adapter *adapter,
 696                         struct crypto_device_info *dev_info,
 697                         int32_t queue_pair_id,
 698                         uint8_t add)
 699 {
 700         struct crypto_queue_pair_info *qp_info;
 701         int enabled;
 702         uint16_t i;
 703
 704         if (dev_info->qpairs == NULL)
 705                 return;
 706
 707         if (queue_pair_id == -1) {
 708                 for (i = 0; i < dev_info->dev->data->nb_queue_pairs; i++)
 709                         eca_update_qp_info(adapter, dev_info, i, add);
 710         } else {
 711                 qp_info = &dev_info->qpairs[queue_pair_id];
 712                 enabled = qp_info->qp_enabled;
 713                 if (add) {
 714                         adapter->nb_qps += !enabled;
 715                         dev_info->num_qpairs += !enabled;
 716                 } else {
 717                         adapter->nb_qps -= enabled;
 718                         dev_info->num_qpairs -= enabled;
 719                 }
 720                 qp_info->qp_enabled = !!add;
 721         }
 722 }
 723
 724 static int
 725 eca_add_queue_pair(struct rte_event_crypto_adapter *adapter,
 726                 uint8_t cdev_id,
 727                 int queue_pair_id)
 728 {
 729         struct crypto_device_info *dev_info = &adapter->cdevs[cdev_id];
 730         struct crypto_queue_pair_info *qpairs;
 731         uint32_t i;
 732
 733         if (dev_info->qpairs == NULL) {
 734                 dev_info->qpairs =
 735                     rte_zmalloc_socket(adapter->mem_name,
 736                                         dev_info->dev->data->nb_queue_pairs *
 737                                         sizeof(struct crypto_queue_pair_info),
 738                                         0, adapter->socket_id);
 739                 if (dev_info->qpairs == NULL)
 740                         return -ENOMEM;
 741
 742                 qpairs = dev_info->qpairs;
 743                 qpairs->op_buffer = rte_zmalloc_socket(adapter->mem_name,
 744                                         BATCH_SIZE *
 745                                         sizeof(struct rte_crypto_op *),
 746                                         0, adapter->socket_id);
 747                 if (!qpairs->op_buffer) {
 748                         rte_free(qpairs);
 749                         return -ENOMEM;
 750                 }
 751         }
 752
 753         if (queue_pair_id == -1) {
 754                 for (i = 0; i < dev_info->dev->data->nb_queue_pairs; i++)
 755                         eca_update_qp_info(adapter, dev_info, i, 1);
 756         } else
 757                 eca_update_qp_info(adapter, dev_info,
 758                                         (uint16_t)queue_pair_id, 1);
 759
 760         return 0;
 761 }
 762
 763 int __rte_experimental
 764 rte_event_crypto_adapter_queue_pair_add(uint8_t id,
 765                         uint8_t cdev_id,
 766                         int32_t queue_pair_id,
 767                         const struct rte_event *event)
 768 {
 769         struct rte_event_crypto_adapter *adapter;
 770         struct rte_eventdev *dev;
 771         struct crypto_device_info *dev_info;
 772         uint32_t cap;
 773         int ret;
 774
 775         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 776
 777         if (!rte_cryptodev_pmd_is_valid_dev(cdev_id)) {
 778                 RTE_EDEV_LOG_ERR("Invalid dev_id=%" PRIu8, cdev_id);
 779                 return -EINVAL;
 780         }
 781
 782         adapter = eca_id_to_adapter(id);
 783         if (adapter == NULL)
 784                 return -EINVAL;
 785
 786         dev = &rte_eventdevs[adapter->eventdev_id];
 787         ret = rte_event_crypto_adapter_caps_get(adapter->eventdev_id,
 788                                                 cdev_id,
 789                                                 &cap);
 790         if (ret) {
 791                 RTE_EDEV_LOG_ERR("Failed to get adapter caps dev %" PRIu8
 792                         " cdev %" PRIu8, id, cdev_id);
 793                 return ret;
 794         }
 795
 796         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) &&
 797             (event == NULL)) {
 798                 RTE_EDEV_LOG_ERR("Conf value can not be NULL for dev_id=%u",
 799                                   cdev_id);
 800                 return -EINVAL;
 801         }
 802
 803         dev_info = &adapter->cdevs[cdev_id];
 804
 805         if (queue_pair_id != -1 &&
 806             (uint16_t)queue_pair_id >= dev_info->dev->data->nb_queue_pairs) {
 807                 RTE_EDEV_LOG_ERR("Invalid queue_pair_id %" PRIu16,
 808                                  (uint16_t)queue_pair_id);
 809                 return -EINVAL;
 810         }
 811
 812         /* In case HW cap is RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD,
 813          * no need of service core as HW supports event forward capability.
 814          */
 815         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) ||
 816             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND &&
 817              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) ||
 818             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
 819              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)) {
 820                 RTE_FUNC_PTR_OR_ERR_RET(
 821                         *dev->dev_ops->crypto_adapter_queue_pair_add,
 822                         -ENOTSUP);
 823                 if (dev_info->qpairs == NULL) {
 824                         dev_info->qpairs =
 825                             rte_zmalloc_socket(adapter->mem_name,
 826                                         dev_info->dev->data->nb_queue_pairs *
 827                                         sizeof(struct crypto_queue_pair_info),
 828                                         0, adapter->socket_id);
 829                         if (dev_info->qpairs == NULL)
 830                                 return -ENOMEM;
 831                 }
 832
 833                 ret = (*dev->dev_ops->crypto_adapter_queue_pair_add)(dev,
 834                                 dev_info->dev,
 835                                 queue_pair_id,
 836                                 event);
 837                 if (ret)
 838                         return ret;
 839
 840                 else
 841                         eca_update_qp_info(adapter, &adapter->cdevs[cdev_id],
 842                                            queue_pair_id, 1);
 843         }
 844
 845         /* In case HW cap is RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW,
 846          * or SW adapter, initiate services so the application can choose
 847          * which ever way it wants to use the adapter.
 848          * Case 1: RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW
 849          *         Application may wants to use one of below two mode
 850          *          a. OP_FORWARD mode -> HW Dequeue + SW enqueue
 851          *          b. OP_NEW mode -> HW Dequeue
 852          * Case 2: No HW caps, use SW adapter
 853          *          a. OP_FORWARD mode -> SW enqueue & dequeue
 854          *          b. OP_NEW mode -> SW Dequeue
 855          */
 856         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
 857              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_FORWARD) ||
 858              (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW) &&
 859               !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) &&
 860               !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) &&
 861                (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA))) {
 862                 rte_spinlock_lock(&adapter->lock);
 863                 ret = eca_init_service(adapter, id);
 864                 if (ret == 0)
 865                         ret = eca_add_queue_pair(adapter, cdev_id,
 866                                                  queue_pair_id);
 867                 rte_spinlock_unlock(&adapter->lock);
 868
 869                 if (ret)
 870                         return ret;
 871
 872                 rte_service_component_runstate_set(adapter->service_id, 1);
 873         }
 874
 875         return 0;
 876 }
 877
 878 int __rte_experimental
 879 rte_event_crypto_adapter_queue_pair_del(uint8_t id, uint8_t cdev_id,
 880                                         int32_t queue_pair_id)
 881 {
 882         struct rte_event_crypto_adapter *adapter;
 883         struct crypto_device_info *dev_info;
 884         struct rte_eventdev *dev;
 885         int ret;
 886         uint32_t cap;
 887         uint16_t i;
 888
 889         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 890
 891         if (!rte_cryptodev_pmd_is_valid_dev(cdev_id)) {
 892                 RTE_EDEV_LOG_ERR("Invalid dev_id=%" PRIu8, cdev_id);
 893                 return -EINVAL;
 894         }
 895
 896         adapter = eca_id_to_adapter(id);
 897         if (adapter == NULL)
 898                 return -EINVAL;
 899
 900         dev = &rte_eventdevs[adapter->eventdev_id];
 901         ret = rte_event_crypto_adapter_caps_get(adapter->eventdev_id,
 902                                                 cdev_id,
 903                                                 &cap);
 904         if (ret)
 905                 return ret;
 906
 907         dev_info = &adapter->cdevs[cdev_id];
 908
 909         if (queue_pair_id != -1 &&
 910             (uint16_t)queue_pair_id >= dev_info->dev->data->nb_queue_pairs) {
 911                 RTE_EDEV_LOG_ERR("Invalid queue_pair_id %" PRIu16,
 912                                  (uint16_t)queue_pair_id);
 913                 return -EINVAL;
 914         }
 915
 916         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) ||
 917             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
 918              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)) {
 919                 RTE_FUNC_PTR_OR_ERR_RET(
 920                         *dev->dev_ops->crypto_adapter_queue_pair_del,
 921                         -ENOTSUP);
 922                 ret = (*dev->dev_ops->crypto_adapter_queue_pair_del)(dev,
 923                                                 dev_info->dev,
 924                                                 queue_pair_id);
 925                 if (ret == 0) {
 926                         eca_update_qp_info(adapter,
 927                                         &adapter->cdevs[cdev_id],
 928                                         queue_pair_id,
 929                                         0);
 930                         if (dev_info->num_qpairs == 0) {
 931                                 rte_free(dev_info->qpairs);
 932                                 dev_info->qpairs = NULL;
 933                         }
 934                 }
 935         } else {
 936                 if (adapter->nb_qps == 0)
 937                         return 0;
 938
 939                 rte_spinlock_lock(&adapter->lock);
 940                 if (queue_pair_id == -1) {
 941                         for (i = 0; i < dev_info->dev->data->nb_queue_pairs;
 942                                 i++)
 943                                 eca_update_qp_info(adapter, dev_info,
 944                                                         queue_pair_id, 0);
 945                 } else {
 946                         eca_update_qp_info(adapter, dev_info,
 947                                                 (uint16_t)queue_pair_id, 0);
 948                 }
 949
 950                 if (dev_info->num_qpairs == 0) {
 951                         rte_free(dev_info->qpairs);
 952                         dev_info->qpairs = NULL;
 953                 }
 954
 955                 rte_spinlock_unlock(&adapter->lock);
 956                 rte_service_component_runstate_set(adapter->service_id,
 957                                 adapter->nb_qps);
 958         }
 959
 960         return ret;
 961 }
 962
 963 static int
 964 eca_adapter_ctrl(uint8_t id, int start)
 965 {
 966         struct rte_event_crypto_adapter *adapter;
 967         struct crypto_device_info *dev_info;
 968         struct rte_eventdev *dev;
 969         uint32_t i;
 970         int use_service;
 971         int stop = !start;
 972
 973         use_service = 0;
 974         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 975         adapter = eca_id_to_adapter(id);
 976         if (adapter == NULL)
 977                 return -EINVAL;
 978
 979         dev = &rte_eventdevs[adapter->eventdev_id];
 980
 981         for (i = 0; i < rte_cryptodev_count(); i++) {
 982                 dev_info = &adapter->cdevs[i];
 983                 /* if start  check for num queue pairs */
 984                 if (start && !dev_info->num_qpairs)
 985                         continue;
 986                 /* if stop check if dev has been started */
 987                 if (stop && !dev_info->dev_started)
 988                         continue;
 989                 use_service |= !dev_info->internal_event_port;
 990                 dev_info->dev_started = start;
 991                 if (dev_info->internal_event_port == 0)
 992                         continue;
 993                 start ? (*dev->dev_ops->crypto_adapter_start)(dev,
 994                                                 &dev_info->dev[i]) :
 995                         (*dev->dev_ops->crypto_adapter_stop)(dev,
 996                                                 &dev_info->dev[i]);
 997         }
 998
 999         if (use_service)
1000                 rte_service_runstate_set(adapter->service_id, start);
1001
1002         return 0;
1003 }
1004
1005 int __rte_experimental
1006 rte_event_crypto_adapter_start(uint8_t id)
1007 {
1008         struct rte_event_crypto_adapter *adapter;
1009
1010         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1011         adapter = eca_id_to_adapter(id);
1012         if (adapter == NULL)
1013                 return -EINVAL;
1014
1015         return eca_adapter_ctrl(id, 1);
1016 }
1017
1018 int __rte_experimental
1019 rte_event_crypto_adapter_stop(uint8_t id)
1020 {
1021         return eca_adapter_ctrl(id, 0);
1022 }
1023
1024 int __rte_experimental
1025 rte_event_crypto_adapter_stats_get(uint8_t id,
1026                                 struct rte_event_crypto_adapter_stats *stats)
1027 {
1028         struct rte_event_crypto_adapter *adapter;
1029         struct rte_event_crypto_adapter_stats dev_stats_sum = { 0 };
1030         struct rte_event_crypto_adapter_stats dev_stats;
1031         struct rte_eventdev *dev;
1032         struct crypto_device_info *dev_info;
1033         uint32_t i;
1034         int ret;
1035
1036         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1037
1038         adapter = eca_id_to_adapter(id);
1039         if (adapter == NULL || stats == NULL)
1040                 return -EINVAL;
1041
1042         dev = &rte_eventdevs[adapter->eventdev_id];
1043         memset(stats, 0, sizeof(*stats));
1044         for (i = 0; i < rte_cryptodev_count(); i++) {
1045                 dev_info = &adapter->cdevs[i];
1046                 if (dev_info->internal_event_port == 0 ||
1047                         dev->dev_ops->crypto_adapter_stats_get == NULL)
1048                         continue;
1049                 ret = (*dev->dev_ops->crypto_adapter_stats_get)(dev,
1050                                                 dev_info->dev,
1051                                                 &dev_stats);
1052                 if (ret)
1053                         continue;
1054
1055                 dev_stats_sum.crypto_deq_count += dev_stats.crypto_deq_count;
1056                 dev_stats_sum.event_enq_count +=
1057                         dev_stats.event_enq_count;
1058         }
1059
1060         if (adapter->service_inited)
1061                 *stats = adapter->crypto_stats;
1062
1063         stats->crypto_deq_count += dev_stats_sum.crypto_deq_count;
1064         stats->event_enq_count += dev_stats_sum.event_enq_count;
1065
1066         return 0;
1067 }
1068
1069 int __rte_experimental
1070 rte_event_crypto_adapter_stats_reset(uint8_t id)
1071 {
1072         struct rte_event_crypto_adapter *adapter;
1073         struct crypto_device_info *dev_info;
1074         struct rte_eventdev *dev;
1075         uint32_t i;
1076
1077         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1078
1079         adapter = eca_id_to_adapter(id);
1080         if (adapter == NULL)
1081                 return -EINVAL;
1082
1083         dev = &rte_eventdevs[adapter->eventdev_id];
1084         for (i = 0; i < rte_cryptodev_count(); i++) {
1085                 dev_info = &adapter->cdevs[i];
1086                 if (dev_info->internal_event_port == 0 ||
1087                         dev->dev_ops->crypto_adapter_stats_reset == NULL)
1088                         continue;
1089                 (*dev->dev_ops->crypto_adapter_stats_reset)(dev,
1090                                                 dev_info->dev);
1091         }
1092
1093         memset(&adapter->crypto_stats, 0, sizeof(adapter->crypto_stats));
1094         return 0;
1095 }
1096
1097 int __rte_experimental
1098 rte_event_crypto_adapter_service_id_get(uint8_t id, uint32_t *service_id)
1099 {
1100         struct rte_event_crypto_adapter *adapter;
1101
1102         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1103
1104         adapter = eca_id_to_adapter(id);
1105         if (adapter == NULL || service_id == NULL)
1106                 return -EINVAL;
1107
1108         if (adapter->service_inited)
1109                 *service_id = adapter->service_id;
1110
1111         return adapter->service_inited ? 0 : -ESRCH;
1112 }
1113
1114 int __rte_experimental
1115 rte_event_crypto_adapter_event_port_get(uint8_t id, uint8_t *event_port_id)
1116 {
1117         struct rte_event_crypto_adapter *adapter;
1118
1119         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1120
1121         adapter = eca_id_to_adapter(id);
1122         if (adapter == NULL || event_port_id == NULL)
1123                 return -EINVAL;
1124
1125         *event_port_id = adapter->event_port_id;
1126
1127         return 0;
1128 }