drivers/crypto/scheduler/scheduler_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32 #include <rte_common.h>
  33 #include <rte_hexdump.h>
  34 #include <rte_cryptodev.h>
  35 #include <rte_cryptodev_pmd.h>
  36 #include <rte_cryptodev_vdev.h>
  37 #include <rte_vdev.h>
  38 #include <rte_malloc.h>
  39 #include <rte_cpuflags.h>
  40 #include <rte_reorder.h>
  41
  42 #include "rte_cryptodev_scheduler.h"
  43 #include "scheduler_pmd_private.h"
  44
  45 uint8_t cryptodev_driver_id;
  46
  47 struct scheduler_init_params {
  48         struct rte_crypto_vdev_init_params def_p;
  49         uint32_t nb_slaves;
  50         enum rte_cryptodev_scheduler_mode mode;
  51         uint32_t enable_ordering;
  52         uint64_t wcmask;
  53         char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
  54                         [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
  55 };
  56
  57 #define RTE_CRYPTODEV_VDEV_NAME                 ("name")
  58 #define RTE_CRYPTODEV_VDEV_SLAVE                ("slave")
  59 #define RTE_CRYPTODEV_VDEV_MODE                 ("mode")
  60 #define RTE_CRYPTODEV_VDEV_ORDERING             ("ordering")
  61 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG        ("max_nb_queue_pairs")
  62 #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG      ("max_nb_sessions")
  63 #define RTE_CRYPTODEV_VDEV_SOCKET_ID            ("socket_id")
  64 #define RTE_CRYPTODEV_VDEV_COREMASK             ("coremask")
  65 #define RTE_CRYPTODEV_VDEV_CORELIST             ("corelist")
  66
  67 const char *scheduler_valid_params[] = {
  68         RTE_CRYPTODEV_VDEV_NAME,
  69         RTE_CRYPTODEV_VDEV_SLAVE,
  70         RTE_CRYPTODEV_VDEV_MODE,
  71         RTE_CRYPTODEV_VDEV_ORDERING,
  72         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
  73         RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
  74         RTE_CRYPTODEV_VDEV_SOCKET_ID,
  75         RTE_CRYPTODEV_VDEV_COREMASK,
  76         RTE_CRYPTODEV_VDEV_CORELIST
  77 };
  78
  79 struct scheduler_parse_map {
  80         const char *name;
  81         uint32_t val;
  82 };
  83
  84 const struct scheduler_parse_map scheduler_mode_map[] = {
  85         {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
  86                         CDEV_SCHED_MODE_ROUNDROBIN},
  87         {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
  88                         CDEV_SCHED_MODE_PKT_SIZE_DISTR},
  89         {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
  90                         CDEV_SCHED_MODE_FAILOVER},
  91         {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
  92                         CDEV_SCHED_MODE_MULTICORE}
  93 };
  94
  95 const struct scheduler_parse_map scheduler_ordering_map[] = {
  96                 {"enable", 1},
  97                 {"disable", 0}
  98 };
  99
 100 static int
 101 cryptodev_scheduler_create(const char *name,
 102                 struct rte_vdev_device *vdev,
 103                 struct scheduler_init_params *init_params)
 104 {
 105         struct rte_cryptodev *dev;
 106         struct scheduler_ctx *sched_ctx;
 107         uint32_t i;
 108         int ret;
 109
 110         if (init_params->def_p.name[0] == '\0')
 111                 snprintf(init_params->def_p.name,
 112                                 sizeof(init_params->def_p.name),
 113                                 "%s", name);
 114
 115         dev = rte_cryptodev_vdev_pmd_init(init_params->def_p.name,
 116                         sizeof(struct scheduler_ctx),
 117                         init_params->def_p.socket_id,
 118                         vdev);
 119         if (dev == NULL) {
 120                 CS_LOG_ERR("driver %s: failed to create cryptodev vdev",
 121                         name);
 122                 return -EFAULT;
 123         }
 124
 125         dev->driver_id = cryptodev_driver_id;
 126         dev->dev_ops = rte_crypto_scheduler_pmd_ops;
 127
 128         sched_ctx = dev->data->dev_private;
 129         sched_ctx->max_nb_queue_pairs =
 130                         init_params->def_p.max_nb_queue_pairs;
 131
 132         if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
 133                 uint16_t i;
 134
 135                 sched_ctx->nb_wc = 0;
 136
 137                 for (i = 0; i < RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES; i++) {
 138                         if (init_params->wcmask & (1ULL << i)) {
 139                                 sched_ctx->wc_pool[sched_ctx->nb_wc++] = i;
 140                                 RTE_LOG(INFO, PMD,
 141                                         "  Worker core[%u]=%u added\n",
 142                                         sched_ctx->nb_wc-1, i);
 143                         }
 144                 }
 145         }
 146
 147         if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
 148                         init_params->mode < CDEV_SCHED_MODE_COUNT) {
 149                 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
 150                         init_params->mode);
 151                 if (ret < 0) {
 152                         rte_cryptodev_pmd_release_device(dev);
 153                         return ret;
 154                 }
 155
 156                 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 157                         if (scheduler_mode_map[i].val != sched_ctx->mode)
 158                                 continue;
 159
 160                         RTE_LOG(INFO, PMD, "  Scheduling mode = %s\n",
 161                                         scheduler_mode_map[i].name);
 162                         break;
 163                 }
 164         }
 165
 166         sched_ctx->reordering_enabled = init_params->enable_ordering;
 167
 168         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 169                 if (scheduler_ordering_map[i].val !=
 170                                 sched_ctx->reordering_enabled)
 171                         continue;
 172
 173                 RTE_LOG(INFO, PMD, "  Packet ordering = %s\n",
 174                                 scheduler_ordering_map[i].name);
 175
 176                 break;
 177         }
 178
 179         for (i = 0; i < init_params->nb_slaves; i++) {
 180                 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
 181                         rte_zmalloc_socket(
 182                                 NULL,
 183                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
 184                                 SOCKET_ID_ANY);
 185
 186                 if (!sched_ctx->init_slave_names[
 187                                 sched_ctx->nb_init_slaves]) {
 188                         CS_LOG_ERR("driver %s: Insufficient memory",
 189                                         name);
 190                         return -ENOMEM;
 191                 }
 192
 193                 strncpy(sched_ctx->init_slave_names[
 194                                         sched_ctx->nb_init_slaves],
 195                                 init_params->slave_names[i],
 196                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 197
 198                 sched_ctx->nb_init_slaves++;
 199         }
 200
 201         /*
 202          * Initialize capabilities structure as an empty structure,
 203          * in case device information is requested when no slaves are attached
 204          */
 205         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 206                         sizeof(struct rte_cryptodev_capabilities),
 207                         0, SOCKET_ID_ANY);
 208
 209         if (!sched_ctx->capabilities) {
 210                 RTE_LOG(ERR, PMD, "Not enough memory for capability "
 211                                 "information\n");
 212                 return -ENOMEM;
 213         }
 214
 215         return 0;
 216 }
 217
 218 static int
 219 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
 220 {
 221         const char *name;
 222         struct rte_cryptodev *dev;
 223         struct scheduler_ctx *sched_ctx;
 224
 225         if (vdev == NULL)
 226                 return -EINVAL;
 227
 228         name = rte_vdev_device_name(vdev);
 229         dev = rte_cryptodev_pmd_get_named_dev(name);
 230         if (dev == NULL)
 231                 return -EINVAL;
 232
 233         sched_ctx = dev->data->dev_private;
 234
 235         if (sched_ctx->nb_slaves) {
 236                 uint32_t i;
 237
 238                 for (i = 0; i < sched_ctx->nb_slaves; i++)
 239                         rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
 240                                         sched_ctx->slaves[i].dev_id);
 241         }
 242
 243         RTE_LOG(INFO, PMD, "Closing Crypto Scheduler device %s on numa "
 244                 "socket %u\n", name, rte_socket_id());
 245
 246         return 0;
 247 }
 248
 249 /** Parse integer from integer argument */
 250 static int
 251 parse_integer_arg(const char *key __rte_unused,
 252                 const char *value, void *extra_args)
 253 {
 254         int *i = (int *) extra_args;
 255
 256         *i = atoi(value);
 257         if (*i < 0) {
 258                 CS_LOG_ERR("Argument has to be positive.\n");
 259                 return -EINVAL;
 260         }
 261
 262         return 0;
 263 }
 264
 265 /** Parse integer from hexadecimal integer argument */
 266 static int
 267 parse_coremask_arg(const char *key __rte_unused,
 268                 const char *value, void *extra_args)
 269 {
 270         struct scheduler_init_params *params = extra_args;
 271
 272         params->wcmask = strtoull(value, NULL, 16);
 273
 274         return 0;
 275 }
 276
 277 /** Parse integer from list of integers argument */
 278 static int
 279 parse_corelist_arg(const char *key __rte_unused,
 280                 const char *value, void *extra_args)
 281 {
 282         struct scheduler_init_params *params = extra_args;
 283
 284         params->wcmask = 0ULL;
 285
 286         const char *token = value;
 287
 288         while (isdigit(token[0])) {
 289                 char *rval;
 290                 unsigned int core = strtoul(token, &rval, 10);
 291
 292                 params->wcmask |= 1ULL << core;
 293                 token = (const char *)rval;
 294                 if (token[0] == '\0')
 295                         break;
 296                 token++;
 297         }
 298
 299         return 0;
 300 }
 301
 302 /** Parse name */
 303 static int
 304 parse_name_arg(const char *key __rte_unused,
 305                 const char *value, void *extra_args)
 306 {
 307         struct rte_crypto_vdev_init_params *params = extra_args;
 308
 309         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 310                 CS_LOG_ERR("Invalid name %s, should be less than "
 311                                 "%u bytes.\n", value,
 312                                 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 313                 return -EINVAL;
 314         }
 315
 316         strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
 317
 318         return 0;
 319 }
 320
 321 /** Parse slave */
 322 static int
 323 parse_slave_arg(const char *key __rte_unused,
 324                 const char *value, void *extra_args)
 325 {
 326         struct scheduler_init_params *param = extra_args;
 327
 328         if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
 329                 CS_LOG_ERR("Too many slaves.\n");
 330                 return -ENOMEM;
 331         }
 332
 333         strncpy(param->slave_names[param->nb_slaves++], value,
 334                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 335
 336         return 0;
 337 }
 338
 339 static int
 340 parse_mode_arg(const char *key __rte_unused,
 341                 const char *value, void *extra_args)
 342 {
 343         struct scheduler_init_params *param = extra_args;
 344         uint32_t i;
 345
 346         for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 347                 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
 348                         param->mode = (enum rte_cryptodev_scheduler_mode)
 349                                         scheduler_mode_map[i].val;
 350                         break;
 351                 }
 352         }
 353
 354         if (i == RTE_DIM(scheduler_mode_map)) {
 355                 CS_LOG_ERR("Unrecognized input.\n");
 356                 return -EINVAL;
 357         }
 358
 359         return 0;
 360 }
 361
 362 static int
 363 parse_ordering_arg(const char *key __rte_unused,
 364                 const char *value, void *extra_args)
 365 {
 366         struct scheduler_init_params *param = extra_args;
 367         uint32_t i;
 368
 369         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 370                 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
 371                         param->enable_ordering =
 372                                         scheduler_ordering_map[i].val;
 373                         break;
 374                 }
 375         }
 376
 377         if (i == RTE_DIM(scheduler_ordering_map)) {
 378                 CS_LOG_ERR("Unrecognized input.\n");
 379                 return -EINVAL;
 380         }
 381
 382         return 0;
 383 }
 384
 385 static int
 386 scheduler_parse_init_params(struct scheduler_init_params *params,
 387                 const char *input_args)
 388 {
 389         struct rte_kvargs *kvlist = NULL;
 390         int ret = 0;
 391
 392         if (params == NULL)
 393                 return -EINVAL;
 394
 395         if (input_args) {
 396                 kvlist = rte_kvargs_parse(input_args,
 397                                 scheduler_valid_params);
 398                 if (kvlist == NULL)
 399                         return -1;
 400
 401                 ret = rte_kvargs_process(kvlist,
 402                                 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
 403                                 &parse_integer_arg,
 404                                 &params->def_p.max_nb_queue_pairs);
 405                 if (ret < 0)
 406                         goto free_kvlist;
 407
 408                 ret = rte_kvargs_process(kvlist,
 409                                 RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
 410                                 &parse_integer_arg,
 411                                 &params->def_p.max_nb_sessions);
 412                 if (ret < 0)
 413                         goto free_kvlist;
 414
 415                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
 416                                 &parse_integer_arg,
 417                                 &params->def_p.socket_id);
 418                 if (ret < 0)
 419                         goto free_kvlist;
 420
 421                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
 422                                 &parse_coremask_arg,
 423                                 params);
 424                 if (ret < 0)
 425                         goto free_kvlist;
 426
 427                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
 428                                 &parse_corelist_arg,
 429                                 params);
 430                 if (ret < 0)
 431                         goto free_kvlist;
 432
 433                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
 434                                 &parse_name_arg,
 435                                 &params->def_p);
 436                 if (ret < 0)
 437                         goto free_kvlist;
 438
 439                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
 440                                 &parse_slave_arg, params);
 441                 if (ret < 0)
 442                         goto free_kvlist;
 443
 444                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
 445                                 &parse_mode_arg, params);
 446                 if (ret < 0)
 447                         goto free_kvlist;
 448
 449                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
 450                                 &parse_ordering_arg, params);
 451                 if (ret < 0)
 452                         goto free_kvlist;
 453         }
 454
 455 free_kvlist:
 456         rte_kvargs_free(kvlist);
 457         return ret;
 458 }
 459
 460 static int
 461 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
 462 {
 463         struct scheduler_init_params init_params = {
 464                 .def_p = {
 465                         RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS,
 466                         RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
 467                         rte_socket_id(),
 468                         ""
 469                 },
 470                 .nb_slaves = 0,
 471                 .mode = CDEV_SCHED_MODE_NOT_SET,
 472                 .enable_ordering = 0,
 473                 .slave_names = { {0} }
 474         };
 475         const char *name;
 476
 477         name = rte_vdev_device_name(vdev);
 478         if (name == NULL)
 479                 return -EINVAL;
 480
 481         scheduler_parse_init_params(&init_params,
 482                                     rte_vdev_device_args(vdev));
 483
 484         RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n",
 485                         name,
 486                         init_params.def_p.socket_id);
 487         RTE_LOG(INFO, PMD, "  Max number of queue pairs = %d\n",
 488                         init_params.def_p.max_nb_queue_pairs);
 489         RTE_LOG(INFO, PMD, "  Max number of sessions = %d\n",
 490                         init_params.def_p.max_nb_sessions);
 491         if (init_params.def_p.name[0] != '\0')
 492                 RTE_LOG(INFO, PMD, "  User defined name = %s\n",
 493                         init_params.def_p.name);
 494         if (init_params.wcmask != 0)
 495                 RTE_LOG(INFO, PMD, "  workers core mask = %"PRIx64"\n",
 496                         init_params.wcmask);
 497
 498         return cryptodev_scheduler_create(name,
 499                                         vdev,
 500                                         &init_params);
 501 }
 502
 503 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
 504         .probe = cryptodev_scheduler_probe,
 505         .remove = cryptodev_scheduler_remove
 506 };
 507
 508 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
 509         cryptodev_scheduler_pmd_drv);
 510 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
 511         "max_nb_queue_pairs=<int> "
 512         "max_nb_sessions=<int> "
 513         "socket_id=<int> "
 514         "slave=<name>");
 515 RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_scheduler_pmd_drv,
 516                 cryptodev_driver_id);