drivers/crypto/scheduler/rte_cryptodev_scheduler.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32 #include <rte_reorder.h>
  33 #include <rte_cryptodev.h>
  34 #include <rte_cryptodev_pmd.h>
  35 #include <rte_malloc.h>
  36
  37 #include "rte_cryptodev_scheduler.h"
  38 #include "scheduler_pmd_private.h"
  39
  40 /** update the scheduler pmd's capability with attaching device's
  41  *  capability.
  42  *  For each device to be attached, the scheduler's capability should be
  43  *  the common capability set of all slaves
  44  **/
  45 static uint32_t
  46 sync_caps(struct rte_cryptodev_capabilities *caps,
  47                 uint32_t nb_caps,
  48                 const struct rte_cryptodev_capabilities *slave_caps)
  49 {
  50         uint32_t sync_nb_caps = nb_caps, nb_slave_caps = 0;
  51         uint32_t i;
  52
  53         while (slave_caps[nb_slave_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED)
  54                 nb_slave_caps++;
  55
  56         if (nb_caps == 0) {
  57                 rte_memcpy(caps, slave_caps, sizeof(*caps) * nb_slave_caps);
  58                 return nb_slave_caps;
  59         }
  60
  61         for (i = 0; i < sync_nb_caps; i++) {
  62                 struct rte_cryptodev_capabilities *cap = &caps[i];
  63                 uint32_t j;
  64
  65                 for (j = 0; j < nb_slave_caps; j++) {
  66                         const struct rte_cryptodev_capabilities *s_cap =
  67                                         &slave_caps[j];
  68
  69                         if (s_cap->op != cap->op || s_cap->sym.xform_type !=
  70                                         cap->sym.xform_type)
  71                                 continue;
  72
  73                         if (s_cap->sym.xform_type ==
  74                                         RTE_CRYPTO_SYM_XFORM_AUTH) {
  75                                 if (s_cap->sym.auth.algo !=
  76                                                 cap->sym.auth.algo)
  77                                         continue;
  78
  79                                 cap->sym.auth.digest_size.min =
  80                                         s_cap->sym.auth.digest_size.min <
  81                                         cap->sym.auth.digest_size.min ?
  82                                         s_cap->sym.auth.digest_size.min :
  83                                         cap->sym.auth.digest_size.min;
  84                                 cap->sym.auth.digest_size.max =
  85                                         s_cap->sym.auth.digest_size.max <
  86                                         cap->sym.auth.digest_size.max ?
  87                                         s_cap->sym.auth.digest_size.max :
  88                                         cap->sym.auth.digest_size.max;
  89
  90                         }
  91
  92                         if (s_cap->sym.xform_type ==
  93                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
  94                                 if (s_cap->sym.cipher.algo !=
  95                                                 cap->sym.cipher.algo)
  96                                         continue;
  97
  98                         /* no common cap found */
  99                         break;
 100                 }
 101
 102                 if (j < nb_slave_caps)
 103                         continue;
 104
 105                 /* remove a uncommon cap from the array */
 106                 for (j = i; j < sync_nb_caps - 1; j++)
 107                         rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap));
 108
 109                 memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap));
 110                 sync_nb_caps--;
 111         }
 112
 113         return sync_nb_caps;
 114 }
 115
 116 static int
 117 update_scheduler_capability(struct scheduler_ctx *sched_ctx)
 118 {
 119         struct rte_cryptodev_capabilities tmp_caps[256] = { {0} };
 120         uint32_t nb_caps = 0, i;
 121
 122         if (sched_ctx->capabilities) {
 123                 rte_free(sched_ctx->capabilities);
 124                 sched_ctx->capabilities = NULL;
 125         }
 126
 127         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 128                 struct rte_cryptodev_info dev_info;
 129
 130                 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
 131
 132                 nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities);
 133                 if (nb_caps == 0)
 134                         return -1;
 135         }
 136
 137         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 138                         sizeof(struct rte_cryptodev_capabilities) *
 139                         (nb_caps + 1), 0, SOCKET_ID_ANY);
 140         if (!sched_ctx->capabilities)
 141                 return -ENOMEM;
 142
 143         rte_memcpy(sched_ctx->capabilities, tmp_caps,
 144                         sizeof(struct rte_cryptodev_capabilities) * nb_caps);
 145
 146         return 0;
 147 }
 148
 149 static void
 150 update_scheduler_feature_flag(struct rte_cryptodev *dev)
 151 {
 152         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 153         uint32_t i;
 154
 155         dev->feature_flags = 0;
 156
 157         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 158                 struct rte_cryptodev_info dev_info;
 159
 160                 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
 161
 162                 dev->feature_flags |= dev_info.feature_flags;
 163         }
 164 }
 165
 166 static void
 167 update_max_nb_qp(struct scheduler_ctx *sched_ctx)
 168 {
 169         uint32_t i;
 170         uint32_t max_nb_qp;
 171
 172         if (!sched_ctx->nb_slaves)
 173                 return;
 174
 175         max_nb_qp = sched_ctx->nb_slaves ? UINT32_MAX : 0;
 176
 177         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 178                 struct rte_cryptodev_info dev_info;
 179
 180                 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
 181                 max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ?
 182                                 dev_info.max_nb_queue_pairs : max_nb_qp;
 183         }
 184
 185         sched_ctx->max_nb_queue_pairs = max_nb_qp;
 186 }
 187
 188 /** Attach a device to the scheduler. */
 189 int
 190 rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id)
 191 {
 192         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 193         struct scheduler_ctx *sched_ctx;
 194         struct scheduler_slave *slave;
 195         struct rte_cryptodev_info dev_info;
 196         uint32_t i;
 197
 198         if (!dev) {
 199                 CS_LOG_ERR("Operation not supported");
 200                 return -ENOTSUP;
 201         }
 202
 203         if (dev->driver_id != cryptodev_driver_id) {
 204                 CS_LOG_ERR("Operation not supported");
 205                 return -ENOTSUP;
 206         }
 207
 208         if (dev->data->dev_started) {
 209                 CS_LOG_ERR("Illegal operation");
 210                 return -EBUSY;
 211         }
 212
 213         sched_ctx = dev->data->dev_private;
 214         if (sched_ctx->nb_slaves >=
 215                         RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
 216                 CS_LOG_ERR("Too many slaves attached");
 217                 return -ENOMEM;
 218         }
 219
 220         for (i = 0; i < sched_ctx->nb_slaves; i++)
 221                 if (sched_ctx->slaves[i].dev_id == slave_id) {
 222                         CS_LOG_ERR("Slave already added");
 223                         return -ENOTSUP;
 224                 }
 225
 226         slave = &sched_ctx->slaves[sched_ctx->nb_slaves];
 227
 228         rte_cryptodev_info_get(slave_id, &dev_info);
 229
 230         slave->dev_id = slave_id;
 231         slave->driver_id = dev_info.driver_id;
 232         sched_ctx->nb_slaves++;
 233
 234         if (update_scheduler_capability(sched_ctx) < 0) {
 235                 slave->dev_id = 0;
 236                 slave->driver_id = 0;
 237                 sched_ctx->nb_slaves--;
 238
 239                 CS_LOG_ERR("capabilities update failed");
 240                 return -ENOTSUP;
 241         }
 242
 243         update_scheduler_feature_flag(dev);
 244
 245         update_max_nb_qp(sched_ctx);
 246
 247         return 0;
 248 }
 249
 250 int
 251 rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id)
 252 {
 253         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 254         struct scheduler_ctx *sched_ctx;
 255         uint32_t i, slave_pos;
 256
 257         if (!dev) {
 258                 CS_LOG_ERR("Operation not supported");
 259                 return -ENOTSUP;
 260         }
 261
 262         if (dev->driver_id != cryptodev_driver_id) {
 263                 CS_LOG_ERR("Operation not supported");
 264                 return -ENOTSUP;
 265         }
 266
 267         if (dev->data->dev_started) {
 268                 CS_LOG_ERR("Illegal operation");
 269                 return -EBUSY;
 270         }
 271
 272         sched_ctx = dev->data->dev_private;
 273
 274         for (slave_pos = 0; slave_pos < sched_ctx->nb_slaves; slave_pos++)
 275                 if (sched_ctx->slaves[slave_pos].dev_id == slave_id)
 276                         break;
 277         if (slave_pos == sched_ctx->nb_slaves) {
 278                 CS_LOG_ERR("Cannot find slave");
 279                 return -ENOTSUP;
 280         }
 281
 282         if (sched_ctx->ops.slave_detach(dev, slave_id) < 0) {
 283                 CS_LOG_ERR("Failed to detach slave");
 284                 return -ENOTSUP;
 285         }
 286
 287         for (i = slave_pos; i < sched_ctx->nb_slaves - 1; i++) {
 288                 memcpy(&sched_ctx->slaves[i], &sched_ctx->slaves[i+1],
 289                                 sizeof(struct scheduler_slave));
 290         }
 291         memset(&sched_ctx->slaves[sched_ctx->nb_slaves - 1], 0,
 292                         sizeof(struct scheduler_slave));
 293         sched_ctx->nb_slaves--;
 294
 295         if (update_scheduler_capability(sched_ctx) < 0) {
 296                 CS_LOG_ERR("capabilities update failed");
 297                 return -ENOTSUP;
 298         }
 299
 300         update_scheduler_feature_flag(dev);
 301
 302         update_max_nb_qp(sched_ctx);
 303
 304         return 0;
 305 }
 306
 307 int
 308 rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id,
 309                 enum rte_cryptodev_scheduler_mode mode)
 310 {
 311         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 312         struct scheduler_ctx *sched_ctx;
 313
 314         if (!dev) {
 315                 CS_LOG_ERR("Operation not supported");
 316                 return -ENOTSUP;
 317         }
 318
 319         if (dev->driver_id != cryptodev_driver_id) {
 320                 CS_LOG_ERR("Operation not supported");
 321                 return -ENOTSUP;
 322         }
 323
 324         if (dev->data->dev_started) {
 325                 CS_LOG_ERR("Illegal operation");
 326                 return -EBUSY;
 327         }
 328
 329         sched_ctx = dev->data->dev_private;
 330
 331         if (mode == sched_ctx->mode)
 332                 return 0;
 333
 334         switch (mode) {
 335         case CDEV_SCHED_MODE_ROUNDROBIN:
 336                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 337                                 roundrobin_scheduler) < 0) {
 338                         CS_LOG_ERR("Failed to load scheduler");
 339                         return -1;
 340                 }
 341                 break;
 342         case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
 343                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 344                                 pkt_size_based_distr_scheduler) < 0) {
 345                         CS_LOG_ERR("Failed to load scheduler");
 346                         return -1;
 347                 }
 348                 break;
 349         case CDEV_SCHED_MODE_FAILOVER:
 350                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 351                                 failover_scheduler) < 0) {
 352                         CS_LOG_ERR("Failed to load scheduler");
 353                         return -1;
 354                 }
 355                 break;
 356         case CDEV_SCHED_MODE_MULTICORE:
 357                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 358                                 multicore_scheduler) < 0) {
 359                         CS_LOG_ERR("Failed to load scheduler");
 360                         return -1;
 361                 }
 362                 break;
 363         default:
 364                 CS_LOG_ERR("Not yet supported");
 365                 return -ENOTSUP;
 366         }
 367
 368         return 0;
 369 }
 370
 371 enum rte_cryptodev_scheduler_mode
 372 rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id)
 373 {
 374         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 375         struct scheduler_ctx *sched_ctx;
 376
 377         if (!dev) {
 378                 CS_LOG_ERR("Operation not supported");
 379                 return -ENOTSUP;
 380         }
 381
 382         if (dev->driver_id != cryptodev_driver_id) {
 383                 CS_LOG_ERR("Operation not supported");
 384                 return -ENOTSUP;
 385         }
 386
 387         sched_ctx = dev->data->dev_private;
 388
 389         return sched_ctx->mode;
 390 }
 391
 392 int
 393 rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id,
 394                 uint32_t enable_reorder)
 395 {
 396         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 397         struct scheduler_ctx *sched_ctx;
 398
 399         if (!dev) {
 400                 CS_LOG_ERR("Operation not supported");
 401                 return -ENOTSUP;
 402         }
 403
 404         if (dev->driver_id != cryptodev_driver_id) {
 405                 CS_LOG_ERR("Operation not supported");
 406                 return -ENOTSUP;
 407         }
 408
 409         if (dev->data->dev_started) {
 410                 CS_LOG_ERR("Illegal operation");
 411                 return -EBUSY;
 412         }
 413
 414         sched_ctx = dev->data->dev_private;
 415
 416         sched_ctx->reordering_enabled = enable_reorder;
 417
 418         return 0;
 419 }
 420
 421 int
 422 rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id)
 423 {
 424         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 425         struct scheduler_ctx *sched_ctx;
 426
 427         if (!dev) {
 428                 CS_LOG_ERR("Operation not supported");
 429                 return -ENOTSUP;
 430         }
 431
 432         if (dev->driver_id != cryptodev_driver_id) {
 433                 CS_LOG_ERR("Operation not supported");
 434                 return -ENOTSUP;
 435         }
 436
 437         sched_ctx = dev->data->dev_private;
 438
 439         return (int)sched_ctx->reordering_enabled;
 440 }
 441
 442 int
 443 rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id,
 444                 struct rte_cryptodev_scheduler *scheduler) {
 445
 446         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 447         struct scheduler_ctx *sched_ctx;
 448
 449         if (!dev) {
 450                 CS_LOG_ERR("Operation not supported");
 451                 return -ENOTSUP;
 452         }
 453
 454         if (dev->driver_id != cryptodev_driver_id) {
 455                 CS_LOG_ERR("Operation not supported");
 456                 return -ENOTSUP;
 457         }
 458
 459         if (dev->data->dev_started) {
 460                 CS_LOG_ERR("Illegal operation");
 461                 return -EBUSY;
 462         }
 463
 464         sched_ctx = dev->data->dev_private;
 465
 466         if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 467                 CS_LOG_ERR("Invalid name %s, should be less than "
 468                                 "%u bytes.\n", scheduler->name,
 469                                 RTE_CRYPTODEV_NAME_MAX_LEN);
 470                 return -EINVAL;
 471         }
 472         snprintf(sched_ctx->name, sizeof(sched_ctx->name), "%s",
 473                         scheduler->name);
 474
 475         if (strlen(scheduler->description) >
 476                         RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) {
 477                 CS_LOG_ERR("Invalid description %s, should be less than "
 478                                 "%u bytes.\n", scheduler->description,
 479                                 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1);
 480                 return -EINVAL;
 481         }
 482         snprintf(sched_ctx->description, sizeof(sched_ctx->description), "%s",
 483                         scheduler->description);
 484
 485         /* load scheduler instance operations functions */
 486         sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair;
 487         sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx;
 488         sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start;
 489         sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop;
 490         sched_ctx->ops.slave_attach = scheduler->ops->slave_attach;
 491         sched_ctx->ops.slave_detach = scheduler->ops->slave_detach;
 492         sched_ctx->ops.option_set = scheduler->ops->option_set;
 493         sched_ctx->ops.option_get = scheduler->ops->option_get;
 494
 495         if (sched_ctx->private_ctx) {
 496                 rte_free(sched_ctx->private_ctx);
 497                 sched_ctx->private_ctx = NULL;
 498         }
 499
 500         if (sched_ctx->ops.create_private_ctx) {
 501                 int ret = (*sched_ctx->ops.create_private_ctx)(dev);
 502
 503                 if (ret < 0) {
 504                         CS_LOG_ERR("Unable to create scheduler private "
 505                                         "context");
 506                         return ret;
 507                 }
 508         }
 509
 510         sched_ctx->mode = scheduler->mode;
 511
 512         return 0;
 513 }
 514
 515 int
 516 rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves)
 517 {
 518         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 519         struct scheduler_ctx *sched_ctx;
 520         uint32_t nb_slaves = 0;
 521
 522         if (!dev) {
 523                 CS_LOG_ERR("Operation not supported");
 524                 return -ENOTSUP;
 525         }
 526
 527         if (dev->driver_id != cryptodev_driver_id) {
 528                 CS_LOG_ERR("Operation not supported");
 529                 return -ENOTSUP;
 530         }
 531
 532         sched_ctx = dev->data->dev_private;
 533
 534         nb_slaves = sched_ctx->nb_slaves;
 535
 536         if (slaves && nb_slaves) {
 537                 uint32_t i;
 538
 539                 for (i = 0; i < nb_slaves; i++)
 540                         slaves[i] = sched_ctx->slaves[i].dev_id;
 541         }
 542
 543         return (int)nb_slaves;
 544 }
 545
 546 int
 547 rte_cryptodev_scheduler_option_set(uint8_t scheduler_id,
 548                 enum rte_cryptodev_schedule_option_type option_type,
 549                 void *option)
 550 {
 551         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 552         struct scheduler_ctx *sched_ctx;
 553
 554         if (option_type == CDEV_SCHED_OPTION_NOT_SET ||
 555                         option_type >= CDEV_SCHED_OPTION_COUNT) {
 556                 CS_LOG_ERR("Invalid option parameter");
 557                 return -EINVAL;
 558         }
 559
 560         if (!option) {
 561                 CS_LOG_ERR("Invalid option parameter");
 562                 return -EINVAL;
 563         }
 564
 565         if (dev->data->dev_started) {
 566                 CS_LOG_ERR("Illegal operation");
 567                 return -EBUSY;
 568         }
 569
 570         sched_ctx = dev->data->dev_private;
 571
 572         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP);
 573
 574         return (*sched_ctx->ops.option_set)(dev, option_type, option);
 575 }
 576
 577 int
 578 rte_cryptodev_scheduler_option_get(uint8_t scheduler_id,
 579                 enum rte_cryptodev_schedule_option_type option_type,
 580                 void *option)
 581 {
 582         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 583         struct scheduler_ctx *sched_ctx;
 584
 585         if (!dev) {
 586                 CS_LOG_ERR("Operation not supported");
 587                 return -ENOTSUP;
 588         }
 589
 590         if (!option) {
 591                 CS_LOG_ERR("Invalid option parameter");
 592                 return -EINVAL;
 593         }
 594
 595         if (dev->driver_id != cryptodev_driver_id) {
 596                 CS_LOG_ERR("Operation not supported");
 597                 return -ENOTSUP;
 598         }
 599
 600         sched_ctx = dev->data->dev_private;
 601
 602         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP);
 603
 604         return (*sched_ctx->ops.option_get)(dev, option_type, option);
 605 }