drivers/crypto/scheduler/scheduler_pmd_ops.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32 #include <string.h>
  33
  34 #include <rte_common.h>
  35 #include <rte_malloc.h>
  36 #include <rte_dev.h>
  37 #include <rte_cryptodev.h>
  38 #include <rte_cryptodev_pmd.h>
  39 #include <rte_reorder.h>
  40
  41 #include "scheduler_pmd_private.h"
  42
  43 /** attaching the slaves predefined by scheduler's EAL options */
  44 static int
  45 scheduler_attach_init_slave(struct rte_cryptodev *dev)
  46 {
  47         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
  48         uint8_t scheduler_id = dev->data->dev_id;
  49         int i;
  50
  51         for (i = sched_ctx->nb_init_slaves - 1; i >= 0; i--) {
  52                 const char *dev_name = sched_ctx->init_slave_names[i];
  53                 struct rte_cryptodev *slave_dev =
  54                                 rte_cryptodev_pmd_get_named_dev(dev_name);
  55                 int status;
  56
  57                 if (!slave_dev) {
  58                         CS_LOG_ERR("Failed to locate slave dev %s",
  59                                         dev_name);
  60                         return -EINVAL;
  61                 }
  62
  63                 status = rte_cryptodev_scheduler_slave_attach(
  64                                 scheduler_id, slave_dev->data->dev_id);
  65
  66                 if (status < 0) {
  67                         CS_LOG_ERR("Failed to attach slave cryptodev %u",
  68                                         slave_dev->data->dev_id);
  69                         return status;
  70                 }
  71
  72                 CS_LOG_INFO("Scheduler %s attached slave %s\n",
  73                                 dev->data->name,
  74                                 sched_ctx->init_slave_names[i]);
  75
  76                 rte_free(sched_ctx->init_slave_names[i]);
  77                 sched_ctx->init_slave_names[i] = NULL;
  78
  79                 sched_ctx->nb_init_slaves -= 1;
  80         }
  81
  82         return 0;
  83 }
  84 /** Configure device */
  85 static int
  86 scheduler_pmd_config(struct rte_cryptodev *dev,
  87                 struct rte_cryptodev_config *config)
  88 {
  89         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
  90         uint32_t i;
  91         int ret;
  92
  93         /* although scheduler_attach_init_slave presents multiple times,
  94          * there will be only 1 meaningful execution.
  95          */
  96         ret = scheduler_attach_init_slave(dev);
  97         if (ret < 0)
  98                 return ret;
  99
 100         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 101                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 102
 103                 ret = rte_cryptodev_configure(slave_dev_id, config);
 104                 if (ret < 0)
 105                         break;
 106         }
 107
 108         return ret;
 109 }
 110
 111 static int
 112 update_order_ring(struct rte_cryptodev *dev, uint16_t qp_id)
 113 {
 114         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 115         struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
 116
 117         if (sched_ctx->reordering_enabled) {
 118                 char order_ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 119                 uint32_t buff_size = rte_align32pow2(
 120                         sched_ctx->nb_slaves * PER_SLAVE_BUFF_SIZE);
 121
 122                 if (qp_ctx->order_ring) {
 123                         rte_ring_free(qp_ctx->order_ring);
 124                         qp_ctx->order_ring = NULL;
 125                 }
 126
 127                 if (!buff_size)
 128                         return 0;
 129
 130                 if (snprintf(order_ring_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 131                         "%s_rb_%u_%u", RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD),
 132                         dev->data->dev_id, qp_id) < 0) {
 133                         CS_LOG_ERR("failed to create unique reorder buffer "
 134                                         "name");
 135                         return -ENOMEM;
 136                 }
 137
 138                 qp_ctx->order_ring = rte_ring_create(order_ring_name,
 139                                 buff_size, rte_socket_id(),
 140                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
 141                 if (!qp_ctx->order_ring) {
 142                         CS_LOG_ERR("failed to create order ring");
 143                         return -ENOMEM;
 144                 }
 145         } else {
 146                 if (qp_ctx->order_ring) {
 147                         rte_ring_free(qp_ctx->order_ring);
 148                         qp_ctx->order_ring = NULL;
 149                 }
 150         }
 151
 152         return 0;
 153 }
 154
 155 /** Start device */
 156 static int
 157 scheduler_pmd_start(struct rte_cryptodev *dev)
 158 {
 159         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 160         uint32_t i;
 161         int ret;
 162
 163         if (dev->data->dev_started)
 164                 return 0;
 165
 166         /* although scheduler_attach_init_slave presents multiple times,
 167          * there will be only 1 meaningful execution.
 168          */
 169         ret = scheduler_attach_init_slave(dev);
 170         if (ret < 0)
 171                 return ret;
 172
 173         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 174                 ret = update_order_ring(dev, i);
 175                 if (ret < 0) {
 176                         CS_LOG_ERR("Failed to update reorder buffer");
 177                         return ret;
 178                 }
 179         }
 180
 181         if (sched_ctx->mode == CDEV_SCHED_MODE_NOT_SET) {
 182                 CS_LOG_ERR("Scheduler mode is not set");
 183                 return -1;
 184         }
 185
 186         if (!sched_ctx->nb_slaves) {
 187                 CS_LOG_ERR("No slave in the scheduler");
 188                 return -1;
 189         }
 190
 191         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.slave_attach, -ENOTSUP);
 192
 193         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 194                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 195
 196                 if ((*sched_ctx->ops.slave_attach)(dev, slave_dev_id) < 0) {
 197                         CS_LOG_ERR("Failed to attach slave");
 198                         return -ENOTSUP;
 199                 }
 200         }
 201
 202         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.scheduler_start, -ENOTSUP);
 203
 204         if ((*sched_ctx->ops.scheduler_start)(dev) < 0) {
 205                 CS_LOG_ERR("Scheduler start failed");
 206                 return -1;
 207         }
 208
 209         /* start all slaves */
 210         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 211                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 212                 struct rte_cryptodev *slave_dev =
 213                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 214
 215                 ret = (*slave_dev->dev_ops->dev_start)(slave_dev);
 216                 if (ret < 0) {
 217                         CS_LOG_ERR("Failed to start slave dev %u",
 218                                         slave_dev_id);
 219                         return ret;
 220                 }
 221         }
 222
 223         return 0;
 224 }
 225
 226 /** Stop device */
 227 static void
 228 scheduler_pmd_stop(struct rte_cryptodev *dev)
 229 {
 230         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 231         uint32_t i;
 232
 233         if (!dev->data->dev_started)
 234                 return;
 235
 236         /* stop all slaves first */
 237         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 238                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 239                 struct rte_cryptodev *slave_dev =
 240                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 241
 242                 (*slave_dev->dev_ops->dev_stop)(slave_dev);
 243         }
 244
 245         if (*sched_ctx->ops.scheduler_stop)
 246                 (*sched_ctx->ops.scheduler_stop)(dev);
 247
 248         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 249                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 250
 251                 if (*sched_ctx->ops.slave_detach)
 252                         (*sched_ctx->ops.slave_detach)(dev, slave_dev_id);
 253         }
 254 }
 255
 256 /** Close device */
 257 static int
 258 scheduler_pmd_close(struct rte_cryptodev *dev)
 259 {
 260         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 261         uint32_t i;
 262         int ret;
 263
 264         /* the dev should be stopped before being closed */
 265         if (dev->data->dev_started)
 266                 return -EBUSY;
 267
 268         /* close all slaves first */
 269         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 270                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 271                 struct rte_cryptodev *slave_dev =
 272                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 273
 274                 ret = (*slave_dev->dev_ops->dev_close)(slave_dev);
 275                 if (ret < 0)
 276                         return ret;
 277         }
 278
 279         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 280                 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i];
 281
 282                 if (qp_ctx->order_ring) {
 283                         rte_ring_free(qp_ctx->order_ring);
 284                         qp_ctx->order_ring = NULL;
 285                 }
 286
 287                 if (qp_ctx->private_qp_ctx) {
 288                         rte_free(qp_ctx->private_qp_ctx);
 289                         qp_ctx->private_qp_ctx = NULL;
 290                 }
 291         }
 292
 293         if (sched_ctx->private_ctx) {
 294                 rte_free(sched_ctx->private_ctx);
 295                 sched_ctx->private_ctx = NULL;
 296         }
 297
 298         if (sched_ctx->capabilities) {
 299                 rte_free(sched_ctx->capabilities);
 300                 sched_ctx->capabilities = NULL;
 301         }
 302
 303         return 0;
 304 }
 305
 306 /** Get device statistics */
 307 static void
 308 scheduler_pmd_stats_get(struct rte_cryptodev *dev,
 309         struct rte_cryptodev_stats *stats)
 310 {
 311         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 312         uint32_t i;
 313
 314         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 315                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 316                 struct rte_cryptodev *slave_dev =
 317                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 318                 struct rte_cryptodev_stats slave_stats = {0};
 319
 320                 (*slave_dev->dev_ops->stats_get)(slave_dev, &slave_stats);
 321
 322                 stats->enqueued_count += slave_stats.enqueued_count;
 323                 stats->dequeued_count += slave_stats.dequeued_count;
 324
 325                 stats->enqueue_err_count += slave_stats.enqueue_err_count;
 326                 stats->dequeue_err_count += slave_stats.dequeue_err_count;
 327         }
 328 }
 329
 330 /** Reset device statistics */
 331 static void
 332 scheduler_pmd_stats_reset(struct rte_cryptodev *dev)
 333 {
 334         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 335         uint32_t i;
 336
 337         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 338                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 339                 struct rte_cryptodev *slave_dev =
 340                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 341
 342                 (*slave_dev->dev_ops->stats_reset)(slave_dev);
 343         }
 344 }
 345
 346 /** Get device info */
 347 static void
 348 scheduler_pmd_info_get(struct rte_cryptodev *dev,
 349                 struct rte_cryptodev_info *dev_info)
 350 {
 351         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 352         uint32_t max_nb_sessions = sched_ctx->nb_slaves ?
 353                         UINT32_MAX : RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS;
 354         uint32_t i;
 355
 356         if (!dev_info)
 357                 return;
 358
 359         /* although scheduler_attach_init_slave presents multiple times,
 360          * there will be only 1 meaningful execution.
 361          */
 362         scheduler_attach_init_slave(dev);
 363
 364         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 365                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 366                 struct rte_cryptodev_info slave_info;
 367
 368                 rte_cryptodev_info_get(slave_dev_id, &slave_info);
 369                 max_nb_sessions = slave_info.sym.max_nb_sessions <
 370                                 max_nb_sessions ?
 371                                 slave_info.sym.max_nb_sessions :
 372                                 max_nb_sessions;
 373         }
 374
 375         dev_info->driver_id = dev->driver_id;
 376         dev_info->feature_flags = dev->feature_flags;
 377         dev_info->capabilities = sched_ctx->capabilities;
 378         dev_info->max_nb_queue_pairs = sched_ctx->max_nb_queue_pairs;
 379         dev_info->sym.max_nb_sessions = max_nb_sessions;
 380 }
 381
 382 /** Release queue pair */
 383 static int
 384 scheduler_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 385 {
 386         struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
 387
 388         if (!qp_ctx)
 389                 return 0;
 390
 391         if (qp_ctx->order_ring)
 392                 rte_ring_free(qp_ctx->order_ring);
 393         if (qp_ctx->private_qp_ctx)
 394                 rte_free(qp_ctx->private_qp_ctx);
 395
 396         rte_free(qp_ctx);
 397         dev->data->queue_pairs[qp_id] = NULL;
 398
 399         return 0;
 400 }
 401
 402 /** Setup a queue pair */
 403 static int
 404 scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 405         const struct rte_cryptodev_qp_conf *qp_conf, int socket_id,
 406         struct rte_mempool *session_pool)
 407 {
 408         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 409         struct scheduler_qp_ctx *qp_ctx;
 410         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 411         uint32_t i;
 412         int ret;
 413
 414         if (snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN,
 415                         "CRYTO_SCHE PMD %u QP %u",
 416                         dev->data->dev_id, qp_id) < 0) {
 417                 CS_LOG_ERR("Failed to create unique queue pair name");
 418                 return -EFAULT;
 419         }
 420
 421         /* Free memory prior to re-allocation if needed. */
 422         if (dev->data->queue_pairs[qp_id] != NULL)
 423                 scheduler_pmd_qp_release(dev, qp_id);
 424
 425         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 426                 uint8_t slave_id = sched_ctx->slaves[i].dev_id;
 427
 428                 /*
 429                  * All slaves will share the same session mempool
 430                  * for session-less operations, so the objects
 431                  * must be big enough for all the drivers used.
 432                  */
 433                 ret = rte_cryptodev_queue_pair_setup(slave_id, qp_id,
 434                                 qp_conf, socket_id, session_pool);
 435                 if (ret < 0)
 436                         return ret;
 437         }
 438
 439         /* Allocate the queue pair data structure. */
 440         qp_ctx = rte_zmalloc_socket(name, sizeof(*qp_ctx), RTE_CACHE_LINE_SIZE,
 441                         socket_id);
 442         if (qp_ctx == NULL)
 443                 return -ENOMEM;
 444
 445         /* The actual available object number = nb_descriptors - 1 */
 446         qp_ctx->max_nb_objs = qp_conf->nb_descriptors - 1;
 447
 448         dev->data->queue_pairs[qp_id] = qp_ctx;
 449
 450         /* although scheduler_attach_init_slave presents multiple times,
 451          * there will be only 1 meaningful execution.
 452          */
 453         ret = scheduler_attach_init_slave(dev);
 454         if (ret < 0) {
 455                 CS_LOG_ERR("Failed to attach slave");
 456                 scheduler_pmd_qp_release(dev, qp_id);
 457                 return ret;
 458         }
 459
 460         if (*sched_ctx->ops.config_queue_pair) {
 461                 if ((*sched_ctx->ops.config_queue_pair)(dev, qp_id) < 0) {
 462                         CS_LOG_ERR("Unable to configure queue pair");
 463                         return -1;
 464                 }
 465         }
 466
 467         return 0;
 468 }
 469
 470 /** Start queue pair */
 471 static int
 472 scheduler_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
 473                 __rte_unused uint16_t queue_pair_id)
 474 {
 475         return -ENOTSUP;
 476 }
 477
 478 /** Stop queue pair */
 479 static int
 480 scheduler_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
 481                 __rte_unused uint16_t queue_pair_id)
 482 {
 483         return -ENOTSUP;
 484 }
 485
 486 /** Return the number of allocated queue pairs */
 487 static uint32_t
 488 scheduler_pmd_qp_count(struct rte_cryptodev *dev)
 489 {
 490         return dev->data->nb_queue_pairs;
 491 }
 492
 493 static uint32_t
 494 scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
 495 {
 496         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 497         uint8_t i = 0;
 498         uint32_t max_priv_sess_size = 0;
 499
 500         /* Check what is the maximum private session size for all slaves */
 501         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 502                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 503                 struct rte_cryptodev *dev = &rte_cryptodevs[slave_dev_id];
 504                 uint32_t priv_sess_size = (*dev->dev_ops->session_get_size)(dev);
 505
 506                 if (max_priv_sess_size < priv_sess_size)
 507                         max_priv_sess_size = priv_sess_size;
 508         }
 509
 510         return max_priv_sess_size;
 511 }
 512
 513 static int
 514 scheduler_pmd_session_configure(struct rte_cryptodev *dev,
 515         struct rte_crypto_sym_xform *xform,
 516         struct rte_cryptodev_sym_session *sess,
 517         struct rte_mempool *mempool)
 518 {
 519         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 520         uint32_t i;
 521         int ret;
 522
 523         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 524                 struct scheduler_slave *slave = &sched_ctx->slaves[i];
 525
 526                 ret = rte_cryptodev_sym_session_init(slave->dev_id, sess,
 527                                         xform, mempool);
 528                 if (ret < 0) {
 529                         CS_LOG_ERR("unabled to config sym session");
 530                         return ret;
 531                 }
 532         }
 533
 534         return 0;
 535 }
 536
 537 /** Clear the memory of session so it doesn't leave key material behind */
 538 static void
 539 scheduler_pmd_session_clear(struct rte_cryptodev *dev,
 540                 struct rte_cryptodev_sym_session *sess)
 541 {
 542         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 543         uint32_t i;
 544
 545         /* Clear private data of slaves */
 546         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 547                 struct scheduler_slave *slave = &sched_ctx->slaves[i];
 548
 549                 rte_cryptodev_sym_session_clear(slave->dev_id, sess);
 550         }
 551 }
 552
 553 struct rte_cryptodev_ops scheduler_pmd_ops = {
 554                 .dev_configure          = scheduler_pmd_config,
 555                 .dev_start              = scheduler_pmd_start,
 556                 .dev_stop               = scheduler_pmd_stop,
 557                 .dev_close              = scheduler_pmd_close,
 558
 559                 .stats_get              = scheduler_pmd_stats_get,
 560                 .stats_reset            = scheduler_pmd_stats_reset,
 561
 562                 .dev_infos_get          = scheduler_pmd_info_get,
 563
 564                 .queue_pair_setup       = scheduler_pmd_qp_setup,
 565                 .queue_pair_release     = scheduler_pmd_qp_release,
 566                 .queue_pair_start       = scheduler_pmd_qp_start,
 567                 .queue_pair_stop        = scheduler_pmd_qp_stop,
 568                 .queue_pair_count       = scheduler_pmd_qp_count,
 569
 570                 .session_get_size       = scheduler_pmd_session_get_size,
 571                 .session_configure      = scheduler_pmd_session_configure,
 572                 .session_clear          = scheduler_pmd_session_clear,
 573 };
 574
 575 struct rte_cryptodev_ops *rte_crypto_scheduler_pmd_ops = &scheduler_pmd_ops;