drivers/crypto/scheduler/scheduler_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32 #include <rte_common.h>
  33 #include <rte_hexdump.h>
  34 #include <rte_cryptodev.h>
  35 #include <rte_cryptodev_pmd.h>
  36 #include <rte_bus_vdev.h>
  37 #include <rte_malloc.h>
  38 #include <rte_cpuflags.h>
  39 #include <rte_reorder.h>
  40
  41 #include "rte_cryptodev_scheduler.h"
  42 #include "scheduler_pmd_private.h"
  43
  44 uint8_t cryptodev_driver_id;
  45
  46 struct scheduler_init_params {
  47         struct rte_cryptodev_pmd_init_params def_p;
  48         uint32_t nb_slaves;
  49         enum rte_cryptodev_scheduler_mode mode;
  50         uint32_t enable_ordering;
  51         uint16_t wc_pool[RTE_MAX_LCORE];
  52         uint16_t nb_wc;
  53         char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
  54                         [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
  55 };
  56
  57 #define RTE_CRYPTODEV_VDEV_NAME                 ("name")
  58 #define RTE_CRYPTODEV_VDEV_SLAVE                ("slave")
  59 #define RTE_CRYPTODEV_VDEV_MODE                 ("mode")
  60 #define RTE_CRYPTODEV_VDEV_ORDERING             ("ordering")
  61 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG        ("max_nb_queue_pairs")
  62 #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG      ("max_nb_sessions")
  63 #define RTE_CRYPTODEV_VDEV_SOCKET_ID            ("socket_id")
  64 #define RTE_CRYPTODEV_VDEV_COREMASK             ("coremask")
  65 #define RTE_CRYPTODEV_VDEV_CORELIST             ("corelist")
  66
  67 const char *scheduler_valid_params[] = {
  68         RTE_CRYPTODEV_VDEV_NAME,
  69         RTE_CRYPTODEV_VDEV_SLAVE,
  70         RTE_CRYPTODEV_VDEV_MODE,
  71         RTE_CRYPTODEV_VDEV_ORDERING,
  72         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
  73         RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
  74         RTE_CRYPTODEV_VDEV_SOCKET_ID,
  75         RTE_CRYPTODEV_VDEV_COREMASK,
  76         RTE_CRYPTODEV_VDEV_CORELIST
  77 };
  78
  79 struct scheduler_parse_map {
  80         const char *name;
  81         uint32_t val;
  82 };
  83
  84 const struct scheduler_parse_map scheduler_mode_map[] = {
  85         {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
  86                         CDEV_SCHED_MODE_ROUNDROBIN},
  87         {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
  88                         CDEV_SCHED_MODE_PKT_SIZE_DISTR},
  89         {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
  90                         CDEV_SCHED_MODE_FAILOVER},
  91         {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
  92                         CDEV_SCHED_MODE_MULTICORE}
  93 };
  94
  95 const struct scheduler_parse_map scheduler_ordering_map[] = {
  96                 {"enable", 1},
  97                 {"disable", 0}
  98 };
  99
 100 static int
 101 cryptodev_scheduler_create(const char *name,
 102                 struct rte_vdev_device *vdev,
 103                 struct scheduler_init_params *init_params)
 104 {
 105         struct rte_cryptodev *dev;
 106         struct scheduler_ctx *sched_ctx;
 107         uint32_t i;
 108         int ret;
 109
 110         dev = rte_cryptodev_pmd_create(name, &vdev->device,
 111                         &init_params->def_p);
 112         if (dev == NULL) {
 113                 CS_LOG_ERR("driver %s: failed to create cryptodev vdev",
 114                         name);
 115                 return -EFAULT;
 116         }
 117
 118         dev->driver_id = cryptodev_driver_id;
 119         dev->dev_ops = rte_crypto_scheduler_pmd_ops;
 120
 121         sched_ctx = dev->data->dev_private;
 122         sched_ctx->max_nb_queue_pairs =
 123                         init_params->def_p.max_nb_queue_pairs;
 124
 125         if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
 126                 uint16_t i;
 127
 128                 sched_ctx->nb_wc = init_params->nb_wc;
 129
 130                 for (i = 0; i < sched_ctx->nb_wc; i++) {
 131                         sched_ctx->wc_pool[i] = init_params->wc_pool[i];
 132                         RTE_LOG(INFO, PMD, "  Worker core[%u]=%u added\n",
 133                                 i, sched_ctx->wc_pool[i]);
 134                 }
 135         }
 136
 137         if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
 138                         init_params->mode < CDEV_SCHED_MODE_COUNT) {
 139                 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
 140                         init_params->mode);
 141                 if (ret < 0) {
 142                         rte_cryptodev_pmd_release_device(dev);
 143                         return ret;
 144                 }
 145
 146                 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 147                         if (scheduler_mode_map[i].val != sched_ctx->mode)
 148                                 continue;
 149
 150                         RTE_LOG(INFO, PMD, "  Scheduling mode = %s\n",
 151                                         scheduler_mode_map[i].name);
 152                         break;
 153                 }
 154         }
 155
 156         sched_ctx->reordering_enabled = init_params->enable_ordering;
 157
 158         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 159                 if (scheduler_ordering_map[i].val !=
 160                                 sched_ctx->reordering_enabled)
 161                         continue;
 162
 163                 RTE_LOG(INFO, PMD, "  Packet ordering = %s\n",
 164                                 scheduler_ordering_map[i].name);
 165
 166                 break;
 167         }
 168
 169         for (i = 0; i < init_params->nb_slaves; i++) {
 170                 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
 171                         rte_zmalloc_socket(
 172                                 NULL,
 173                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
 174                                 SOCKET_ID_ANY);
 175
 176                 if (!sched_ctx->init_slave_names[
 177                                 sched_ctx->nb_init_slaves]) {
 178                         CS_LOG_ERR("driver %s: Insufficient memory",
 179                                         name);
 180                         return -ENOMEM;
 181                 }
 182
 183                 strncpy(sched_ctx->init_slave_names[
 184                                         sched_ctx->nb_init_slaves],
 185                                 init_params->slave_names[i],
 186                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 187
 188                 sched_ctx->nb_init_slaves++;
 189         }
 190
 191         /*
 192          * Initialize capabilities structure as an empty structure,
 193          * in case device information is requested when no slaves are attached
 194          */
 195         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 196                         sizeof(struct rte_cryptodev_capabilities),
 197                         0, SOCKET_ID_ANY);
 198
 199         if (!sched_ctx->capabilities) {
 200                 RTE_LOG(ERR, PMD, "Not enough memory for capability "
 201                                 "information\n");
 202                 return -ENOMEM;
 203         }
 204
 205         return 0;
 206 }
 207
 208 static int
 209 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
 210 {
 211         const char *name;
 212         struct rte_cryptodev *dev;
 213         struct scheduler_ctx *sched_ctx;
 214
 215         if (vdev == NULL)
 216                 return -EINVAL;
 217
 218         name = rte_vdev_device_name(vdev);
 219         dev = rte_cryptodev_pmd_get_named_dev(name);
 220         if (dev == NULL)
 221                 return -EINVAL;
 222
 223         sched_ctx = dev->data->dev_private;
 224
 225         if (sched_ctx->nb_slaves) {
 226                 uint32_t i;
 227
 228                 for (i = 0; i < sched_ctx->nb_slaves; i++)
 229                         rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
 230                                         sched_ctx->slaves[i].dev_id);
 231         }
 232
 233         return rte_cryptodev_pmd_destroy(dev);
 234 }
 235
 236 /** Parse integer from integer argument */
 237 static int
 238 parse_integer_arg(const char *key __rte_unused,
 239                 const char *value, void *extra_args)
 240 {
 241         int *i = (int *) extra_args;
 242
 243         *i = atoi(value);
 244         if (*i < 0) {
 245                 CS_LOG_ERR("Argument has to be positive.\n");
 246                 return -EINVAL;
 247         }
 248
 249         return 0;
 250 }
 251
 252 /** Parse integer from hexadecimal integer argument */
 253 static int
 254 parse_coremask_arg(const char *key __rte_unused,
 255                 const char *value, void *extra_args)
 256 {
 257         int i, j, val;
 258         uint16_t idx = 0;
 259         char c;
 260         struct scheduler_init_params *params = extra_args;
 261
 262         params->nb_wc = 0;
 263
 264         if (value == NULL)
 265                 return -1;
 266         /* Remove all blank characters ahead and after .
 267          * Remove 0x/0X if exists.
 268          */
 269         while (isblank(*value))
 270                 value++;
 271         if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
 272                 value += 2;
 273         i = strlen(value);
 274         while ((i > 0) && isblank(value[i - 1]))
 275                 i--;
 276
 277         if (i == 0)
 278                 return -1;
 279
 280         for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
 281                 c = value[i];
 282                 if (isxdigit(c) == 0) {
 283                         /* invalid characters */
 284                         return -1;
 285                 }
 286                 if (isdigit(c))
 287                         val = c - '0';
 288                 else if (isupper(c))
 289                         val = c - 'A' + 10;
 290                 else
 291                         val = c - 'a' + 10;
 292
 293                 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
 294                         if ((1 << j) & val)
 295                                 params->wc_pool[params->nb_wc++] = idx;
 296                 }
 297         }
 298
 299         return 0;
 300 }
 301
 302 /** Parse integer from list of integers argument */
 303 static int
 304 parse_corelist_arg(const char *key __rte_unused,
 305                 const char *value, void *extra_args)
 306 {
 307         struct scheduler_init_params *params = extra_args;
 308
 309         params->nb_wc = 0;
 310
 311         const char *token = value;
 312
 313         while (isdigit(token[0])) {
 314                 char *rval;
 315                 unsigned int core = strtoul(token, &rval, 10);
 316
 317                 if (core >= RTE_MAX_LCORE) {
 318                         CS_LOG_ERR("Invalid worker core %u, should be smaller "
 319                                    "than %u.\n", core, RTE_MAX_LCORE);
 320                 }
 321                 params->wc_pool[params->nb_wc++] = (uint16_t)core;
 322                 token = (const char *)rval;
 323                 if (token[0] == '\0')
 324                         break;
 325                 token++;
 326         }
 327
 328         return 0;
 329 }
 330
 331 /** Parse name */
 332 static int
 333 parse_name_arg(const char *key __rte_unused,
 334                 const char *value, void *extra_args)
 335 {
 336         struct rte_cryptodev_pmd_init_params *params = extra_args;
 337
 338         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 339                 CS_LOG_ERR("Invalid name %s, should be less than "
 340                                 "%u bytes.\n", value,
 341                                 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 342                 return -EINVAL;
 343         }
 344
 345         strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
 346
 347         return 0;
 348 }
 349
 350 /** Parse slave */
 351 static int
 352 parse_slave_arg(const char *key __rte_unused,
 353                 const char *value, void *extra_args)
 354 {
 355         struct scheduler_init_params *param = extra_args;
 356
 357         if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
 358                 CS_LOG_ERR("Too many slaves.\n");
 359                 return -ENOMEM;
 360         }
 361
 362         strncpy(param->slave_names[param->nb_slaves++], value,
 363                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 364
 365         return 0;
 366 }
 367
 368 static int
 369 parse_mode_arg(const char *key __rte_unused,
 370                 const char *value, void *extra_args)
 371 {
 372         struct scheduler_init_params *param = extra_args;
 373         uint32_t i;
 374
 375         for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 376                 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
 377                         param->mode = (enum rte_cryptodev_scheduler_mode)
 378                                         scheduler_mode_map[i].val;
 379                         break;
 380                 }
 381         }
 382
 383         if (i == RTE_DIM(scheduler_mode_map)) {
 384                 CS_LOG_ERR("Unrecognized input.\n");
 385                 return -EINVAL;
 386         }
 387
 388         return 0;
 389 }
 390
 391 static int
 392 parse_ordering_arg(const char *key __rte_unused,
 393                 const char *value, void *extra_args)
 394 {
 395         struct scheduler_init_params *param = extra_args;
 396         uint32_t i;
 397
 398         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 399                 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
 400                         param->enable_ordering =
 401                                         scheduler_ordering_map[i].val;
 402                         break;
 403                 }
 404         }
 405
 406         if (i == RTE_DIM(scheduler_ordering_map)) {
 407                 CS_LOG_ERR("Unrecognized input.\n");
 408                 return -EINVAL;
 409         }
 410
 411         return 0;
 412 }
 413
 414 static int
 415 scheduler_parse_init_params(struct scheduler_init_params *params,
 416                 const char *input_args)
 417 {
 418         struct rte_kvargs *kvlist = NULL;
 419         int ret = 0;
 420
 421         if (params == NULL)
 422                 return -EINVAL;
 423
 424         if (input_args) {
 425                 kvlist = rte_kvargs_parse(input_args,
 426                                 scheduler_valid_params);
 427                 if (kvlist == NULL)
 428                         return -1;
 429
 430                 ret = rte_kvargs_process(kvlist,
 431                                 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
 432                                 &parse_integer_arg,
 433                                 &params->def_p.max_nb_queue_pairs);
 434                 if (ret < 0)
 435                         goto free_kvlist;
 436
 437                 ret = rte_kvargs_process(kvlist,
 438                                 RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
 439                                 &parse_integer_arg,
 440                                 &params->def_p.max_nb_sessions);
 441                 if (ret < 0)
 442                         goto free_kvlist;
 443
 444                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
 445                                 &parse_integer_arg,
 446                                 &params->def_p.socket_id);
 447                 if (ret < 0)
 448                         goto free_kvlist;
 449
 450                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
 451                                 &parse_coremask_arg,
 452                                 params);
 453                 if (ret < 0)
 454                         goto free_kvlist;
 455
 456                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
 457                                 &parse_corelist_arg,
 458                                 params);
 459                 if (ret < 0)
 460                         goto free_kvlist;
 461
 462                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
 463                                 &parse_name_arg,
 464                                 &params->def_p);
 465                 if (ret < 0)
 466                         goto free_kvlist;
 467
 468                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
 469                                 &parse_slave_arg, params);
 470                 if (ret < 0)
 471                         goto free_kvlist;
 472
 473                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
 474                                 &parse_mode_arg, params);
 475                 if (ret < 0)
 476                         goto free_kvlist;
 477
 478                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
 479                                 &parse_ordering_arg, params);
 480                 if (ret < 0)
 481                         goto free_kvlist;
 482         }
 483
 484 free_kvlist:
 485         rte_kvargs_free(kvlist);
 486         return ret;
 487 }
 488
 489 static int
 490 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
 491 {
 492         struct scheduler_init_params init_params = {
 493                 .def_p = {
 494                         "",
 495                         sizeof(struct scheduler_ctx),
 496                         rte_socket_id(),
 497                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 498                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
 499                 },
 500                 .nb_slaves = 0,
 501                 .mode = CDEV_SCHED_MODE_NOT_SET,
 502                 .enable_ordering = 0,
 503                 .slave_names = { {0} }
 504         };
 505         const char *name;
 506
 507         name = rte_vdev_device_name(vdev);
 508         if (name == NULL)
 509                 return -EINVAL;
 510
 511         scheduler_parse_init_params(&init_params,
 512                                     rte_vdev_device_args(vdev));
 513
 514
 515         return cryptodev_scheduler_create(name,
 516                                         vdev,
 517                                         &init_params);
 518 }
 519
 520 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
 521         .probe = cryptodev_scheduler_probe,
 522         .remove = cryptodev_scheduler_remove
 523 };
 524
 525 static struct cryptodev_driver scheduler_crypto_drv;
 526
 527 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
 528         cryptodev_scheduler_pmd_drv);
 529 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
 530         "max_nb_queue_pairs=<int> "
 531         "max_nb_sessions=<int> "
 532         "socket_id=<int> "
 533         "slave=<name>");
 534 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
 535                 cryptodev_scheduler_pmd_drv,
 536                 cryptodev_driver_id);