drivers/crypto/scheduler/rte_cryptodev_scheduler.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32 #include <rte_reorder.h>
  33 #include <rte_cryptodev.h>
  34 #include <rte_cryptodev_pmd.h>
  35 #include <rte_malloc.h>
  36
  37 #include "rte_cryptodev_scheduler.h"
  38 #include "scheduler_pmd_private.h"
  39
  40 /** update the scheduler pmd's capability with attaching device's
  41  *  capability.
  42  *  For each device to be attached, the scheduler's capability should be
  43  *  the common capability set of all slaves
  44  **/
  45 static uint32_t
  46 sync_caps(struct rte_cryptodev_capabilities *caps,
  47                 uint32_t nb_caps,
  48                 const struct rte_cryptodev_capabilities *slave_caps)
  49 {
  50         uint32_t sync_nb_caps = nb_caps, nb_slave_caps = 0;
  51         uint32_t i;
  52
  53         while (slave_caps[nb_slave_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED)
  54                 nb_slave_caps++;
  55
  56         if (nb_caps == 0) {
  57                 rte_memcpy(caps, slave_caps, sizeof(*caps) * nb_slave_caps);
  58                 return nb_slave_caps;
  59         }
  60
  61         for (i = 0; i < sync_nb_caps; i++) {
  62                 struct rte_cryptodev_capabilities *cap = &caps[i];
  63                 uint32_t j;
  64
  65                 for (j = 0; j < nb_slave_caps; j++) {
  66                         const struct rte_cryptodev_capabilities *s_cap =
  67                                         &slave_caps[j];
  68
  69                         if (s_cap->op != cap->op || s_cap->sym.xform_type !=
  70                                         cap->sym.xform_type)
  71                                 continue;
  72
  73                         if (s_cap->sym.xform_type ==
  74                                         RTE_CRYPTO_SYM_XFORM_AUTH) {
  75                                 if (s_cap->sym.auth.algo !=
  76                                                 cap->sym.auth.algo)
  77                                         continue;
  78
  79                                 cap->sym.auth.digest_size.min =
  80                                         s_cap->sym.auth.digest_size.min <
  81                                         cap->sym.auth.digest_size.min ?
  82                                         s_cap->sym.auth.digest_size.min :
  83                                         cap->sym.auth.digest_size.min;
  84                                 cap->sym.auth.digest_size.max =
  85                                         s_cap->sym.auth.digest_size.max <
  86                                         cap->sym.auth.digest_size.max ?
  87                                         s_cap->sym.auth.digest_size.max :
  88                                         cap->sym.auth.digest_size.max;
  89
  90                         }
  91
  92                         if (s_cap->sym.xform_type ==
  93                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
  94                                 if (s_cap->sym.cipher.algo !=
  95                                                 cap->sym.cipher.algo)
  96                                         continue;
  97
  98                         /* no common cap found */
  99                         break;
 100                 }
 101
 102                 if (j < nb_slave_caps)
 103                         continue;
 104
 105                 /* remove a uncommon cap from the array */
 106                 for (j = i; j < sync_nb_caps - 1; j++)
 107                         rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap));
 108
 109                 memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap));
 110                 sync_nb_caps--;
 111         }
 112
 113         return sync_nb_caps;
 114 }
 115
 116 static int
 117 update_scheduler_capability(struct scheduler_ctx *sched_ctx)
 118 {
 119         struct rte_cryptodev_capabilities tmp_caps[256] = { {0} };
 120         uint32_t nb_caps = 0, i;
 121
 122         if (sched_ctx->capabilities)
 123                 rte_free(sched_ctx->capabilities);
 124
 125         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 126                 struct rte_cryptodev_info dev_info;
 127
 128                 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
 129
 130                 nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities);
 131                 if (nb_caps == 0)
 132                         return -1;
 133         }
 134
 135         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 136                         sizeof(struct rte_cryptodev_capabilities) *
 137                         (nb_caps + 1), 0, SOCKET_ID_ANY);
 138         if (!sched_ctx->capabilities)
 139                 return -ENOMEM;
 140
 141         rte_memcpy(sched_ctx->capabilities, tmp_caps,
 142                         sizeof(struct rte_cryptodev_capabilities) * nb_caps);
 143
 144         return 0;
 145 }
 146
 147 static void
 148 update_scheduler_feature_flag(struct rte_cryptodev *dev)
 149 {
 150         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 151         uint32_t i;
 152
 153         dev->feature_flags = 0;
 154
 155         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 156                 struct rte_cryptodev_info dev_info;
 157
 158                 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
 159
 160                 dev->feature_flags |= dev_info.feature_flags;
 161         }
 162 }
 163
 164 static void
 165 update_max_nb_qp(struct scheduler_ctx *sched_ctx)
 166 {
 167         uint32_t i;
 168         uint32_t max_nb_qp;
 169
 170         if (!sched_ctx->nb_slaves)
 171                 return;
 172
 173         max_nb_qp = sched_ctx->nb_slaves ? UINT32_MAX : 0;
 174
 175         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 176                 struct rte_cryptodev_info dev_info;
 177
 178                 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
 179                 max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ?
 180                                 dev_info.max_nb_queue_pairs : max_nb_qp;
 181         }
 182
 183         sched_ctx->max_nb_queue_pairs = max_nb_qp;
 184 }
 185
 186 /** Attach a device to the scheduler. */
 187 int
 188 rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id)
 189 {
 190         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 191         struct scheduler_ctx *sched_ctx;
 192         struct scheduler_slave *slave;
 193         struct rte_cryptodev_info dev_info;
 194         uint32_t i;
 195
 196         if (!dev) {
 197                 CS_LOG_ERR("Operation not supported");
 198                 return -ENOTSUP;
 199         }
 200
 201         if (dev->driver_id != cryptodev_driver_id) {
 202                 CS_LOG_ERR("Operation not supported");
 203                 return -ENOTSUP;
 204         }
 205
 206         if (dev->data->dev_started) {
 207                 CS_LOG_ERR("Illegal operation");
 208                 return -EBUSY;
 209         }
 210
 211         sched_ctx = dev->data->dev_private;
 212         if (sched_ctx->nb_slaves >=
 213                         RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
 214                 CS_LOG_ERR("Too many slaves attached");
 215                 return -ENOMEM;
 216         }
 217
 218         for (i = 0; i < sched_ctx->nb_slaves; i++)
 219                 if (sched_ctx->slaves[i].dev_id == slave_id) {
 220                         CS_LOG_ERR("Slave already added");
 221                         return -ENOTSUP;
 222                 }
 223
 224         slave = &sched_ctx->slaves[sched_ctx->nb_slaves];
 225
 226         rte_cryptodev_info_get(slave_id, &dev_info);
 227
 228         slave->dev_id = slave_id;
 229         slave->driver_id = dev_info.driver_id;
 230         sched_ctx->nb_slaves++;
 231
 232         if (update_scheduler_capability(sched_ctx) < 0) {
 233                 slave->dev_id = 0;
 234                 slave->driver_id = 0;
 235                 sched_ctx->nb_slaves--;
 236
 237                 CS_LOG_ERR("capabilities update failed");
 238                 return -ENOTSUP;
 239         }
 240
 241         update_scheduler_feature_flag(dev);
 242
 243         update_max_nb_qp(sched_ctx);
 244
 245         return 0;
 246 }
 247
 248 int
 249 rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id)
 250 {
 251         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 252         struct scheduler_ctx *sched_ctx;
 253         uint32_t i, slave_pos;
 254
 255         if (!dev) {
 256                 CS_LOG_ERR("Operation not supported");
 257                 return -ENOTSUP;
 258         }
 259
 260         if (dev->driver_id != cryptodev_driver_id) {
 261                 CS_LOG_ERR("Operation not supported");
 262                 return -ENOTSUP;
 263         }
 264
 265         if (dev->data->dev_started) {
 266                 CS_LOG_ERR("Illegal operation");
 267                 return -EBUSY;
 268         }
 269
 270         sched_ctx = dev->data->dev_private;
 271
 272         for (slave_pos = 0; slave_pos < sched_ctx->nb_slaves; slave_pos++)
 273                 if (sched_ctx->slaves[slave_pos].dev_id == slave_id)
 274                         break;
 275         if (slave_pos == sched_ctx->nb_slaves) {
 276                 CS_LOG_ERR("Cannot find slave");
 277                 return -ENOTSUP;
 278         }
 279
 280         if (sched_ctx->ops.slave_detach(dev, slave_id) < 0) {
 281                 CS_LOG_ERR("Failed to detach slave");
 282                 return -ENOTSUP;
 283         }
 284
 285         for (i = slave_pos; i < sched_ctx->nb_slaves - 1; i++) {
 286                 memcpy(&sched_ctx->slaves[i], &sched_ctx->slaves[i+1],
 287                                 sizeof(struct scheduler_slave));
 288         }
 289         memset(&sched_ctx->slaves[sched_ctx->nb_slaves - 1], 0,
 290                         sizeof(struct scheduler_slave));
 291         sched_ctx->nb_slaves--;
 292
 293         if (update_scheduler_capability(sched_ctx) < 0) {
 294                 CS_LOG_ERR("capabilities update failed");
 295                 return -ENOTSUP;
 296         }
 297
 298         update_scheduler_feature_flag(dev);
 299
 300         update_max_nb_qp(sched_ctx);
 301
 302         return 0;
 303 }
 304
 305 int
 306 rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id,
 307                 enum rte_cryptodev_scheduler_mode mode)
 308 {
 309         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 310         struct scheduler_ctx *sched_ctx;
 311
 312         if (!dev) {
 313                 CS_LOG_ERR("Operation not supported");
 314                 return -ENOTSUP;
 315         }
 316
 317         if (dev->driver_id != cryptodev_driver_id) {
 318                 CS_LOG_ERR("Operation not supported");
 319                 return -ENOTSUP;
 320         }
 321
 322         if (dev->data->dev_started) {
 323                 CS_LOG_ERR("Illegal operation");
 324                 return -EBUSY;
 325         }
 326
 327         sched_ctx = dev->data->dev_private;
 328
 329         if (mode == sched_ctx->mode)
 330                 return 0;
 331
 332         switch (mode) {
 333         case CDEV_SCHED_MODE_ROUNDROBIN:
 334                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 335                                 roundrobin_scheduler) < 0) {
 336                         CS_LOG_ERR("Failed to load scheduler");
 337                         return -1;
 338                 }
 339                 break;
 340         case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
 341                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 342                                 pkt_size_based_distr_scheduler) < 0) {
 343                         CS_LOG_ERR("Failed to load scheduler");
 344                         return -1;
 345                 }
 346                 break;
 347         case CDEV_SCHED_MODE_FAILOVER:
 348                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 349                                 failover_scheduler) < 0) {
 350                         CS_LOG_ERR("Failed to load scheduler");
 351                         return -1;
 352                 }
 353                 break;
 354         case CDEV_SCHED_MODE_MULTICORE:
 355                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 356                                 multicore_scheduler) < 0) {
 357                         CS_LOG_ERR("Failed to load scheduler");
 358                         return -1;
 359                 }
 360                 break;
 361         default:
 362                 CS_LOG_ERR("Not yet supported");
 363                 return -ENOTSUP;
 364         }
 365
 366         return 0;
 367 }
 368
 369 enum rte_cryptodev_scheduler_mode
 370 rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id)
 371 {
 372         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 373         struct scheduler_ctx *sched_ctx;
 374
 375         if (!dev) {
 376                 CS_LOG_ERR("Operation not supported");
 377                 return -ENOTSUP;
 378         }
 379
 380         if (dev->driver_id != cryptodev_driver_id) {
 381                 CS_LOG_ERR("Operation not supported");
 382                 return -ENOTSUP;
 383         }
 384
 385         sched_ctx = dev->data->dev_private;
 386
 387         return sched_ctx->mode;
 388 }
 389
 390 int
 391 rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id,
 392                 uint32_t enable_reorder)
 393 {
 394         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 395         struct scheduler_ctx *sched_ctx;
 396
 397         if (!dev) {
 398                 CS_LOG_ERR("Operation not supported");
 399                 return -ENOTSUP;
 400         }
 401
 402         if (dev->driver_id != cryptodev_driver_id) {
 403                 CS_LOG_ERR("Operation not supported");
 404                 return -ENOTSUP;
 405         }
 406
 407         if (dev->data->dev_started) {
 408                 CS_LOG_ERR("Illegal operation");
 409                 return -EBUSY;
 410         }
 411
 412         sched_ctx = dev->data->dev_private;
 413
 414         sched_ctx->reordering_enabled = enable_reorder;
 415
 416         return 0;
 417 }
 418
 419 int
 420 rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id)
 421 {
 422         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 423         struct scheduler_ctx *sched_ctx;
 424
 425         if (!dev) {
 426                 CS_LOG_ERR("Operation not supported");
 427                 return -ENOTSUP;
 428         }
 429
 430         if (dev->driver_id != cryptodev_driver_id) {
 431                 CS_LOG_ERR("Operation not supported");
 432                 return -ENOTSUP;
 433         }
 434
 435         sched_ctx = dev->data->dev_private;
 436
 437         return (int)sched_ctx->reordering_enabled;
 438 }
 439
 440 int
 441 rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id,
 442                 struct rte_cryptodev_scheduler *scheduler) {
 443
 444         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 445         struct scheduler_ctx *sched_ctx;
 446
 447         if (!dev) {
 448                 CS_LOG_ERR("Operation not supported");
 449                 return -ENOTSUP;
 450         }
 451
 452         if (dev->driver_id != cryptodev_driver_id) {
 453                 CS_LOG_ERR("Operation not supported");
 454                 return -ENOTSUP;
 455         }
 456
 457         if (dev->data->dev_started) {
 458                 CS_LOG_ERR("Illegal operation");
 459                 return -EBUSY;
 460         }
 461
 462         sched_ctx = dev->data->dev_private;
 463
 464         if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 465                 CS_LOG_ERR("Invalid name %s, should be less than "
 466                                 "%u bytes.\n", scheduler->name,
 467                                 RTE_CRYPTODEV_NAME_MAX_LEN);
 468                 return -EINVAL;
 469         }
 470         snprintf(sched_ctx->name, sizeof(sched_ctx->name), "%s",
 471                         scheduler->name);
 472
 473         if (strlen(scheduler->description) >
 474                         RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) {
 475                 CS_LOG_ERR("Invalid description %s, should be less than "
 476                                 "%u bytes.\n", scheduler->description,
 477                                 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1);
 478                 return -EINVAL;
 479         }
 480         snprintf(sched_ctx->description, sizeof(sched_ctx->description), "%s",
 481                         scheduler->description);
 482
 483         /* load scheduler instance operations functions */
 484         sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair;
 485         sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx;
 486         sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start;
 487         sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop;
 488         sched_ctx->ops.slave_attach = scheduler->ops->slave_attach;
 489         sched_ctx->ops.slave_detach = scheduler->ops->slave_detach;
 490         sched_ctx->ops.option_set = scheduler->ops->option_set;
 491         sched_ctx->ops.option_get = scheduler->ops->option_get;
 492
 493         if (sched_ctx->private_ctx)
 494                 rte_free(sched_ctx->private_ctx);
 495
 496         if (sched_ctx->ops.create_private_ctx) {
 497                 int ret = (*sched_ctx->ops.create_private_ctx)(dev);
 498
 499                 if (ret < 0) {
 500                         CS_LOG_ERR("Unable to create scheduler private "
 501                                         "context");
 502                         return ret;
 503                 }
 504         }
 505
 506         sched_ctx->mode = scheduler->mode;
 507
 508         return 0;
 509 }
 510
 511 int
 512 rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves)
 513 {
 514         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 515         struct scheduler_ctx *sched_ctx;
 516         uint32_t nb_slaves = 0;
 517
 518         if (!dev) {
 519                 CS_LOG_ERR("Operation not supported");
 520                 return -ENOTSUP;
 521         }
 522
 523         if (dev->driver_id != cryptodev_driver_id) {
 524                 CS_LOG_ERR("Operation not supported");
 525                 return -ENOTSUP;
 526         }
 527
 528         sched_ctx = dev->data->dev_private;
 529
 530         nb_slaves = sched_ctx->nb_slaves;
 531
 532         if (slaves && nb_slaves) {
 533                 uint32_t i;
 534
 535                 for (i = 0; i < nb_slaves; i++)
 536                         slaves[i] = sched_ctx->slaves[i].dev_id;
 537         }
 538
 539         return (int)nb_slaves;
 540 }
 541
 542 int
 543 rte_cryptodev_scheduler_option_set(uint8_t scheduler_id,
 544                 enum rte_cryptodev_schedule_option_type option_type,
 545                 void *option)
 546 {
 547         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 548         struct scheduler_ctx *sched_ctx;
 549
 550         if (option_type == CDEV_SCHED_OPTION_NOT_SET ||
 551                         option_type >= CDEV_SCHED_OPTION_COUNT) {
 552                 CS_LOG_ERR("Invalid option parameter");
 553                 return -EINVAL;
 554         }
 555
 556         if (!option) {
 557                 CS_LOG_ERR("Invalid option parameter");
 558                 return -EINVAL;
 559         }
 560
 561         if (dev->data->dev_started) {
 562                 CS_LOG_ERR("Illegal operation");
 563                 return -EBUSY;
 564         }
 565
 566         sched_ctx = dev->data->dev_private;
 567
 568         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP);
 569
 570         return (*sched_ctx->ops.option_set)(dev, option_type, option);
 571 }
 572
 573 int
 574 rte_cryptodev_scheduler_option_get(uint8_t scheduler_id,
 575                 enum rte_cryptodev_schedule_option_type option_type,
 576                 void *option)
 577 {
 578         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 579         struct scheduler_ctx *sched_ctx;
 580
 581         if (!dev) {
 582                 CS_LOG_ERR("Operation not supported");
 583                 return -ENOTSUP;
 584         }
 585
 586         if (!option) {
 587                 CS_LOG_ERR("Invalid option parameter");
 588                 return -EINVAL;
 589         }
 590
 591         if (dev->driver_id != cryptodev_driver_id) {
 592                 CS_LOG_ERR("Operation not supported");
 593                 return -ENOTSUP;
 594         }
 595
 596         sched_ctx = dev->data->dev_private;
 597
 598         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP);
 599
 600         return (*sched_ctx->ops.option_get)(dev, option_type, option);
 601 }