drivers/crypto/scheduler/scheduler_pmd_ops.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32 #include <string.h>
  33
  34 #include <rte_common.h>
  35 #include <rte_malloc.h>
  36 #include <rte_dev.h>
  37 #include <rte_cryptodev.h>
  38 #include <rte_cryptodev_pmd.h>
  39 #include <rte_reorder.h>
  40
  41 #include "scheduler_pmd_private.h"
  42
  43 /** attaching the slaves predefined by scheduler's EAL options */
  44 static int
  45 scheduler_attach_init_slave(struct rte_cryptodev *dev)
  46 {
  47         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
  48         uint8_t scheduler_id = dev->data->dev_id;
  49         int i;
  50
  51         for (i = sched_ctx->nb_init_slaves - 1; i >= 0; i--) {
  52                 const char *dev_name = sched_ctx->init_slave_names[i];
  53                 struct rte_cryptodev *slave_dev =
  54                                 rte_cryptodev_pmd_get_named_dev(dev_name);
  55                 int status;
  56
  57                 if (!slave_dev) {
  58                         CS_LOG_ERR("Failed to locate slave dev %s",
  59                                         dev_name);
  60                         return -EINVAL;
  61                 }
  62
  63                 status = rte_cryptodev_scheduler_slave_attach(
  64                                 scheduler_id, slave_dev->data->dev_id);
  65
  66                 if (status < 0) {
  67                         CS_LOG_ERR("Failed to attach slave cryptodev %u",
  68                                         slave_dev->data->dev_id);
  69                         return status;
  70                 }
  71
  72                 CS_LOG_INFO("Scheduler %s attached slave %s\n",
  73                                 dev->data->name,
  74                                 sched_ctx->init_slave_names[i]);
  75
  76                 rte_free(sched_ctx->init_slave_names[i]);
  77
  78                 sched_ctx->nb_init_slaves -= 1;
  79         }
  80
  81         return 0;
  82 }
  83 /** Configure device */
  84 static int
  85 scheduler_pmd_config(struct rte_cryptodev *dev,
  86                 struct rte_cryptodev_config *config)
  87 {
  88         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
  89         uint32_t i;
  90         int ret;
  91
  92         /* although scheduler_attach_init_slave presents multiple times,
  93          * there will be only 1 meaningful execution.
  94          */
  95         ret = scheduler_attach_init_slave(dev);
  96         if (ret < 0)
  97                 return ret;
  98
  99         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 100                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 101
 102                 ret = rte_cryptodev_configure(slave_dev_id, config);
 103                 if (ret < 0)
 104                         break;
 105         }
 106
 107         return ret;
 108 }
 109
 110 static int
 111 update_order_ring(struct rte_cryptodev *dev, uint16_t qp_id)
 112 {
 113         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 114         struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
 115
 116         if (sched_ctx->reordering_enabled) {
 117                 char order_ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 118                 uint32_t buff_size = rte_align32pow2(
 119                         sched_ctx->nb_slaves * PER_SLAVE_BUFF_SIZE);
 120
 121                 if (qp_ctx->order_ring) {
 122                         rte_ring_free(qp_ctx->order_ring);
 123                         qp_ctx->order_ring = NULL;
 124                 }
 125
 126                 if (!buff_size)
 127                         return 0;
 128
 129                 if (snprintf(order_ring_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 130                         "%s_rb_%u_%u", RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD),
 131                         dev->data->dev_id, qp_id) < 0) {
 132                         CS_LOG_ERR("failed to create unique reorder buffer "
 133                                         "name");
 134                         return -ENOMEM;
 135                 }
 136
 137                 qp_ctx->order_ring = rte_ring_create(order_ring_name,
 138                                 buff_size, rte_socket_id(),
 139                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
 140                 if (!qp_ctx->order_ring) {
 141                         CS_LOG_ERR("failed to create order ring");
 142                         return -ENOMEM;
 143                 }
 144         } else {
 145                 if (qp_ctx->order_ring) {
 146                         rte_ring_free(qp_ctx->order_ring);
 147                         qp_ctx->order_ring = NULL;
 148                 }
 149         }
 150
 151         return 0;
 152 }
 153
 154 /** Start device */
 155 static int
 156 scheduler_pmd_start(struct rte_cryptodev *dev)
 157 {
 158         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 159         uint32_t i;
 160         int ret;
 161
 162         if (dev->data->dev_started)
 163                 return 0;
 164
 165         /* although scheduler_attach_init_slave presents multiple times,
 166          * there will be only 1 meaningful execution.
 167          */
 168         ret = scheduler_attach_init_slave(dev);
 169         if (ret < 0)
 170                 return ret;
 171
 172         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 173                 ret = update_order_ring(dev, i);
 174                 if (ret < 0) {
 175                         CS_LOG_ERR("Failed to update reorder buffer");
 176                         return ret;
 177                 }
 178         }
 179
 180         if (sched_ctx->mode == CDEV_SCHED_MODE_NOT_SET) {
 181                 CS_LOG_ERR("Scheduler mode is not set");
 182                 return -1;
 183         }
 184
 185         if (!sched_ctx->nb_slaves) {
 186                 CS_LOG_ERR("No slave in the scheduler");
 187                 return -1;
 188         }
 189
 190         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.slave_attach, -ENOTSUP);
 191
 192         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 193                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 194
 195                 if ((*sched_ctx->ops.slave_attach)(dev, slave_dev_id) < 0) {
 196                         CS_LOG_ERR("Failed to attach slave");
 197                         return -ENOTSUP;
 198                 }
 199         }
 200
 201         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.scheduler_start, -ENOTSUP);
 202
 203         if ((*sched_ctx->ops.scheduler_start)(dev) < 0) {
 204                 CS_LOG_ERR("Scheduler start failed");
 205                 return -1;
 206         }
 207
 208         /* start all slaves */
 209         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 210                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 211                 struct rte_cryptodev *slave_dev =
 212                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 213
 214                 ret = (*slave_dev->dev_ops->dev_start)(slave_dev);
 215                 if (ret < 0) {
 216                         CS_LOG_ERR("Failed to start slave dev %u",
 217                                         slave_dev_id);
 218                         return ret;
 219                 }
 220         }
 221
 222         return 0;
 223 }
 224
 225 /** Stop device */
 226 static void
 227 scheduler_pmd_stop(struct rte_cryptodev *dev)
 228 {
 229         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 230         uint32_t i;
 231
 232         if (!dev->data->dev_started)
 233                 return;
 234
 235         /* stop all slaves first */
 236         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 237                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 238                 struct rte_cryptodev *slave_dev =
 239                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 240
 241                 (*slave_dev->dev_ops->dev_stop)(slave_dev);
 242         }
 243
 244         if (*sched_ctx->ops.scheduler_stop)
 245                 (*sched_ctx->ops.scheduler_stop)(dev);
 246
 247         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 248                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 249
 250                 if (*sched_ctx->ops.slave_detach)
 251                         (*sched_ctx->ops.slave_detach)(dev, slave_dev_id);
 252         }
 253 }
 254
 255 /** Close device */
 256 static int
 257 scheduler_pmd_close(struct rte_cryptodev *dev)
 258 {
 259         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 260         uint32_t i;
 261         int ret;
 262
 263         /* the dev should be stopped before being closed */
 264         if (dev->data->dev_started)
 265                 return -EBUSY;
 266
 267         /* close all slaves first */
 268         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 269                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 270                 struct rte_cryptodev *slave_dev =
 271                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 272
 273                 ret = (*slave_dev->dev_ops->dev_close)(slave_dev);
 274                 if (ret < 0)
 275                         return ret;
 276         }
 277
 278         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 279                 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i];
 280
 281                 if (qp_ctx->order_ring) {
 282                         rte_ring_free(qp_ctx->order_ring);
 283                         qp_ctx->order_ring = NULL;
 284                 }
 285
 286                 if (qp_ctx->private_qp_ctx) {
 287                         rte_free(qp_ctx->private_qp_ctx);
 288                         qp_ctx->private_qp_ctx = NULL;
 289                 }
 290         }
 291
 292         if (sched_ctx->private_ctx)
 293                 rte_free(sched_ctx->private_ctx);
 294
 295         if (sched_ctx->capabilities)
 296                 rte_free(sched_ctx->capabilities);
 297
 298         return 0;
 299 }
 300
 301 /** Get device statistics */
 302 static void
 303 scheduler_pmd_stats_get(struct rte_cryptodev *dev,
 304         struct rte_cryptodev_stats *stats)
 305 {
 306         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 307         uint32_t i;
 308
 309         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 310                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 311                 struct rte_cryptodev *slave_dev =
 312                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 313                 struct rte_cryptodev_stats slave_stats = {0};
 314
 315                 (*slave_dev->dev_ops->stats_get)(slave_dev, &slave_stats);
 316
 317                 stats->enqueued_count += slave_stats.enqueued_count;
 318                 stats->dequeued_count += slave_stats.dequeued_count;
 319
 320                 stats->enqueue_err_count += slave_stats.enqueue_err_count;
 321                 stats->dequeue_err_count += slave_stats.dequeue_err_count;
 322         }
 323 }
 324
 325 /** Reset device statistics */
 326 static void
 327 scheduler_pmd_stats_reset(struct rte_cryptodev *dev)
 328 {
 329         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 330         uint32_t i;
 331
 332         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 333                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 334                 struct rte_cryptodev *slave_dev =
 335                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 336
 337                 (*slave_dev->dev_ops->stats_reset)(slave_dev);
 338         }
 339 }
 340
 341 /** Get device info */
 342 static void
 343 scheduler_pmd_info_get(struct rte_cryptodev *dev,
 344                 struct rte_cryptodev_info *dev_info)
 345 {
 346         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 347         uint32_t max_nb_sessions = sched_ctx->nb_slaves ?
 348                         UINT32_MAX : RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS;
 349         uint32_t i;
 350
 351         if (!dev_info)
 352                 return;
 353
 354         /* although scheduler_attach_init_slave presents multiple times,
 355          * there will be only 1 meaningful execution.
 356          */
 357         scheduler_attach_init_slave(dev);
 358
 359         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 360                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 361                 struct rte_cryptodev_info slave_info;
 362
 363                 rte_cryptodev_info_get(slave_dev_id, &slave_info);
 364                 max_nb_sessions = slave_info.sym.max_nb_sessions <
 365                                 max_nb_sessions ?
 366                                 slave_info.sym.max_nb_sessions :
 367                                 max_nb_sessions;
 368         }
 369
 370         dev_info->driver_id = dev->driver_id;
 371         dev_info->feature_flags = dev->feature_flags;
 372         dev_info->capabilities = sched_ctx->capabilities;
 373         dev_info->max_nb_queue_pairs = sched_ctx->max_nb_queue_pairs;
 374         dev_info->sym.max_nb_sessions = max_nb_sessions;
 375 }
 376
 377 /** Release queue pair */
 378 static int
 379 scheduler_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 380 {
 381         struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
 382
 383         if (!qp_ctx)
 384                 return 0;
 385
 386         if (qp_ctx->order_ring)
 387                 rte_ring_free(qp_ctx->order_ring);
 388         if (qp_ctx->private_qp_ctx)
 389                 rte_free(qp_ctx->private_qp_ctx);
 390
 391         rte_free(qp_ctx);
 392         dev->data->queue_pairs[qp_id] = NULL;
 393
 394         return 0;
 395 }
 396
 397 /** Setup a queue pair */
 398 static int
 399 scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 400         const struct rte_cryptodev_qp_conf *qp_conf, int socket_id,
 401         struct rte_mempool *session_pool)
 402 {
 403         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 404         struct scheduler_qp_ctx *qp_ctx;
 405         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 406         uint32_t i;
 407         int ret;
 408
 409         if (snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN,
 410                         "CRYTO_SCHE PMD %u QP %u",
 411                         dev->data->dev_id, qp_id) < 0) {
 412                 CS_LOG_ERR("Failed to create unique queue pair name");
 413                 return -EFAULT;
 414         }
 415
 416         /* Free memory prior to re-allocation if needed. */
 417         if (dev->data->queue_pairs[qp_id] != NULL)
 418                 scheduler_pmd_qp_release(dev, qp_id);
 419
 420         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 421                 uint8_t slave_id = sched_ctx->slaves[i].dev_id;
 422
 423                 /*
 424                  * All slaves will share the same session mempool
 425                  * for session-less operations, so the objects
 426                  * must be big enough for all the drivers used.
 427                  */
 428                 ret = rte_cryptodev_queue_pair_setup(slave_id, qp_id,
 429                                 qp_conf, socket_id, session_pool);
 430                 if (ret < 0)
 431                         return ret;
 432         }
 433
 434         /* Allocate the queue pair data structure. */
 435         qp_ctx = rte_zmalloc_socket(name, sizeof(*qp_ctx), RTE_CACHE_LINE_SIZE,
 436                         socket_id);
 437         if (qp_ctx == NULL)
 438                 return -ENOMEM;
 439
 440         /* The actual available object number = nb_descriptors - 1 */
 441         qp_ctx->max_nb_objs = qp_conf->nb_descriptors - 1;
 442
 443         dev->data->queue_pairs[qp_id] = qp_ctx;
 444
 445         /* although scheduler_attach_init_slave presents multiple times,
 446          * there will be only 1 meaningful execution.
 447          */
 448         ret = scheduler_attach_init_slave(dev);
 449         if (ret < 0) {
 450                 CS_LOG_ERR("Failed to attach slave");
 451                 scheduler_pmd_qp_release(dev, qp_id);
 452                 return ret;
 453         }
 454
 455         if (*sched_ctx->ops.config_queue_pair) {
 456                 if ((*sched_ctx->ops.config_queue_pair)(dev, qp_id) < 0) {
 457                         CS_LOG_ERR("Unable to configure queue pair");
 458                         return -1;
 459                 }
 460         }
 461
 462         return 0;
 463 }
 464
 465 /** Start queue pair */
 466 static int
 467 scheduler_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
 468                 __rte_unused uint16_t queue_pair_id)
 469 {
 470         return -ENOTSUP;
 471 }
 472
 473 /** Stop queue pair */
 474 static int
 475 scheduler_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
 476                 __rte_unused uint16_t queue_pair_id)
 477 {
 478         return -ENOTSUP;
 479 }
 480
 481 /** Return the number of allocated queue pairs */
 482 static uint32_t
 483 scheduler_pmd_qp_count(struct rte_cryptodev *dev)
 484 {
 485         return dev->data->nb_queue_pairs;
 486 }
 487
 488 static uint32_t
 489 scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
 490 {
 491         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 492         uint8_t i = 0;
 493         uint32_t max_priv_sess_size = 0;
 494
 495         /* Check what is the maximum private session size for all slaves */
 496         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 497                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 498                 struct rte_cryptodev *dev = &rte_cryptodevs[slave_dev_id];
 499                 uint32_t priv_sess_size = (*dev->dev_ops->session_get_size)(dev);
 500
 501                 if (max_priv_sess_size < priv_sess_size)
 502                         max_priv_sess_size = priv_sess_size;
 503         }
 504
 505         return max_priv_sess_size;
 506 }
 507
 508 static int
 509 scheduler_pmd_session_configure(struct rte_cryptodev *dev,
 510         struct rte_crypto_sym_xform *xform,
 511         struct rte_cryptodev_sym_session *sess,
 512         struct rte_mempool *mempool)
 513 {
 514         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 515         uint32_t i;
 516         int ret;
 517
 518         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 519                 struct scheduler_slave *slave = &sched_ctx->slaves[i];
 520
 521                 ret = rte_cryptodev_sym_session_init(slave->dev_id, sess,
 522                                         xform, mempool);
 523                 if (ret < 0) {
 524                         CS_LOG_ERR("unabled to config sym session");
 525                         return ret;
 526                 }
 527         }
 528
 529         return 0;
 530 }
 531
 532 /** Clear the memory of session so it doesn't leave key material behind */
 533 static void
 534 scheduler_pmd_session_clear(struct rte_cryptodev *dev,
 535                 struct rte_cryptodev_sym_session *sess)
 536 {
 537         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 538         uint32_t i;
 539
 540         /* Clear private data of slaves */
 541         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 542                 struct scheduler_slave *slave = &sched_ctx->slaves[i];
 543
 544                 rte_cryptodev_sym_session_clear(slave->dev_id, sess);
 545         }
 546 }
 547
 548 struct rte_cryptodev_ops scheduler_pmd_ops = {
 549                 .dev_configure          = scheduler_pmd_config,
 550                 .dev_start              = scheduler_pmd_start,
 551                 .dev_stop               = scheduler_pmd_stop,
 552                 .dev_close              = scheduler_pmd_close,
 553
 554                 .stats_get              = scheduler_pmd_stats_get,
 555                 .stats_reset            = scheduler_pmd_stats_reset,
 556
 557                 .dev_infos_get          = scheduler_pmd_info_get,
 558
 559                 .queue_pair_setup       = scheduler_pmd_qp_setup,
 560                 .queue_pair_release     = scheduler_pmd_qp_release,
 561                 .queue_pair_start       = scheduler_pmd_qp_start,
 562                 .queue_pair_stop        = scheduler_pmd_qp_stop,
 563                 .queue_pair_count       = scheduler_pmd_qp_count,
 564
 565                 .session_get_size       = scheduler_pmd_session_get_size,
 566                 .session_configure      = scheduler_pmd_session_configure,
 567                 .session_clear          = scheduler_pmd_session_clear,
 568 };
 569
 570 struct rte_cryptodev_ops *rte_crypto_scheduler_pmd_ops = &scheduler_pmd_ops;