drivers/crypto/scheduler/scheduler_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32 #include <rte_common.h>
  33 #include <rte_hexdump.h>
  34 #include <rte_cryptodev.h>
  35 #include <rte_cryptodev_pmd.h>
  36 #include <rte_bus_vdev.h>
  37 #include <rte_malloc.h>
  38 #include <rte_cpuflags.h>
  39 #include <rte_reorder.h>
  40 #include <rte_string_fns.h>
  41
  42 #include "rte_cryptodev_scheduler.h"
  43 #include "scheduler_pmd_private.h"
  44
  45 uint8_t cryptodev_driver_id;
  46
  47 struct scheduler_init_params {
  48         struct rte_cryptodev_pmd_init_params def_p;
  49         uint32_t nb_slaves;
  50         enum rte_cryptodev_scheduler_mode mode;
  51         uint32_t enable_ordering;
  52         uint16_t wc_pool[RTE_MAX_LCORE];
  53         uint16_t nb_wc;
  54         char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
  55                         [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
  56 };
  57
  58 #define RTE_CRYPTODEV_VDEV_NAME                 ("name")
  59 #define RTE_CRYPTODEV_VDEV_SLAVE                ("slave")
  60 #define RTE_CRYPTODEV_VDEV_MODE                 ("mode")
  61 #define RTE_CRYPTODEV_VDEV_ORDERING             ("ordering")
  62 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG        ("max_nb_queue_pairs")
  63 #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG      ("max_nb_sessions")
  64 #define RTE_CRYPTODEV_VDEV_SOCKET_ID            ("socket_id")
  65 #define RTE_CRYPTODEV_VDEV_COREMASK             ("coremask")
  66 #define RTE_CRYPTODEV_VDEV_CORELIST             ("corelist")
  67
  68 const char *scheduler_valid_params[] = {
  69         RTE_CRYPTODEV_VDEV_NAME,
  70         RTE_CRYPTODEV_VDEV_SLAVE,
  71         RTE_CRYPTODEV_VDEV_MODE,
  72         RTE_CRYPTODEV_VDEV_ORDERING,
  73         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
  74         RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
  75         RTE_CRYPTODEV_VDEV_SOCKET_ID,
  76         RTE_CRYPTODEV_VDEV_COREMASK,
  77         RTE_CRYPTODEV_VDEV_CORELIST
  78 };
  79
  80 struct scheduler_parse_map {
  81         const char *name;
  82         uint32_t val;
  83 };
  84
  85 const struct scheduler_parse_map scheduler_mode_map[] = {
  86         {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
  87                         CDEV_SCHED_MODE_ROUNDROBIN},
  88         {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
  89                         CDEV_SCHED_MODE_PKT_SIZE_DISTR},
  90         {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
  91                         CDEV_SCHED_MODE_FAILOVER},
  92         {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
  93                         CDEV_SCHED_MODE_MULTICORE}
  94 };
  95
  96 const struct scheduler_parse_map scheduler_ordering_map[] = {
  97                 {"enable", 1},
  98                 {"disable", 0}
  99 };
 100
 101 static int
 102 cryptodev_scheduler_create(const char *name,
 103                 struct rte_vdev_device *vdev,
 104                 struct scheduler_init_params *init_params)
 105 {
 106         struct rte_cryptodev *dev;
 107         struct scheduler_ctx *sched_ctx;
 108         uint32_t i;
 109         int ret;
 110
 111         dev = rte_cryptodev_pmd_create(name, &vdev->device,
 112                         &init_params->def_p);
 113         if (dev == NULL) {
 114                 CS_LOG_ERR("driver %s: failed to create cryptodev vdev",
 115                         name);
 116                 return -EFAULT;
 117         }
 118
 119         dev->driver_id = cryptodev_driver_id;
 120         dev->dev_ops = rte_crypto_scheduler_pmd_ops;
 121
 122         sched_ctx = dev->data->dev_private;
 123         sched_ctx->max_nb_queue_pairs =
 124                         init_params->def_p.max_nb_queue_pairs;
 125
 126         if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
 127                 uint16_t i;
 128
 129                 sched_ctx->nb_wc = init_params->nb_wc;
 130
 131                 for (i = 0; i < sched_ctx->nb_wc; i++) {
 132                         sched_ctx->wc_pool[i] = init_params->wc_pool[i];
 133                         RTE_LOG(INFO, PMD, "  Worker core[%u]=%u added\n",
 134                                 i, sched_ctx->wc_pool[i]);
 135                 }
 136         }
 137
 138         if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
 139                         init_params->mode < CDEV_SCHED_MODE_COUNT) {
 140                 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
 141                         init_params->mode);
 142                 if (ret < 0) {
 143                         rte_cryptodev_pmd_release_device(dev);
 144                         return ret;
 145                 }
 146
 147                 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 148                         if (scheduler_mode_map[i].val != sched_ctx->mode)
 149                                 continue;
 150
 151                         RTE_LOG(INFO, PMD, "  Scheduling mode = %s\n",
 152                                         scheduler_mode_map[i].name);
 153                         break;
 154                 }
 155         }
 156
 157         sched_ctx->reordering_enabled = init_params->enable_ordering;
 158
 159         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 160                 if (scheduler_ordering_map[i].val !=
 161                                 sched_ctx->reordering_enabled)
 162                         continue;
 163
 164                 RTE_LOG(INFO, PMD, "  Packet ordering = %s\n",
 165                                 scheduler_ordering_map[i].name);
 166
 167                 break;
 168         }
 169
 170         for (i = 0; i < init_params->nb_slaves; i++) {
 171                 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
 172                         rte_zmalloc_socket(
 173                                 NULL,
 174                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
 175                                 SOCKET_ID_ANY);
 176
 177                 if (!sched_ctx->init_slave_names[
 178                                 sched_ctx->nb_init_slaves]) {
 179                         CS_LOG_ERR("driver %s: Insufficient memory",
 180                                         name);
 181                         return -ENOMEM;
 182                 }
 183
 184                 strncpy(sched_ctx->init_slave_names[
 185                                         sched_ctx->nb_init_slaves],
 186                                 init_params->slave_names[i],
 187                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 188
 189                 sched_ctx->nb_init_slaves++;
 190         }
 191
 192         /*
 193          * Initialize capabilities structure as an empty structure,
 194          * in case device information is requested when no slaves are attached
 195          */
 196         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 197                         sizeof(struct rte_cryptodev_capabilities),
 198                         0, SOCKET_ID_ANY);
 199
 200         if (!sched_ctx->capabilities) {
 201                 RTE_LOG(ERR, PMD, "Not enough memory for capability "
 202                                 "information\n");
 203                 return -ENOMEM;
 204         }
 205
 206         return 0;
 207 }
 208
 209 static int
 210 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
 211 {
 212         const char *name;
 213         struct rte_cryptodev *dev;
 214         struct scheduler_ctx *sched_ctx;
 215
 216         if (vdev == NULL)
 217                 return -EINVAL;
 218
 219         name = rte_vdev_device_name(vdev);
 220         dev = rte_cryptodev_pmd_get_named_dev(name);
 221         if (dev == NULL)
 222                 return -EINVAL;
 223
 224         sched_ctx = dev->data->dev_private;
 225
 226         if (sched_ctx->nb_slaves) {
 227                 uint32_t i;
 228
 229                 for (i = 0; i < sched_ctx->nb_slaves; i++)
 230                         rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
 231                                         sched_ctx->slaves[i].dev_id);
 232         }
 233
 234         return rte_cryptodev_pmd_destroy(dev);
 235 }
 236
 237 /** Parse integer from integer argument */
 238 static int
 239 parse_integer_arg(const char *key __rte_unused,
 240                 const char *value, void *extra_args)
 241 {
 242         int *i = (int *) extra_args;
 243
 244         *i = atoi(value);
 245         if (*i < 0) {
 246                 CS_LOG_ERR("Argument has to be positive.\n");
 247                 return -EINVAL;
 248         }
 249
 250         return 0;
 251 }
 252
 253 /** Parse integer from hexadecimal integer argument */
 254 static int
 255 parse_coremask_arg(const char *key __rte_unused,
 256                 const char *value, void *extra_args)
 257 {
 258         int i, j, val;
 259         uint16_t idx = 0;
 260         char c;
 261         struct scheduler_init_params *params = extra_args;
 262
 263         params->nb_wc = 0;
 264
 265         if (value == NULL)
 266                 return -1;
 267         /* Remove all blank characters ahead and after .
 268          * Remove 0x/0X if exists.
 269          */
 270         while (isblank(*value))
 271                 value++;
 272         if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
 273                 value += 2;
 274         i = strlen(value);
 275         while ((i > 0) && isblank(value[i - 1]))
 276                 i--;
 277
 278         if (i == 0)
 279                 return -1;
 280
 281         for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
 282                 c = value[i];
 283                 if (isxdigit(c) == 0) {
 284                         /* invalid characters */
 285                         return -1;
 286                 }
 287                 if (isdigit(c))
 288                         val = c - '0';
 289                 else if (isupper(c))
 290                         val = c - 'A' + 10;
 291                 else
 292                         val = c - 'a' + 10;
 293
 294                 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
 295                         if ((1 << j) & val)
 296                                 params->wc_pool[params->nb_wc++] = idx;
 297                 }
 298         }
 299
 300         return 0;
 301 }
 302
 303 /** Parse integer from list of integers argument */
 304 static int
 305 parse_corelist_arg(const char *key __rte_unused,
 306                 const char *value, void *extra_args)
 307 {
 308         struct scheduler_init_params *params = extra_args;
 309
 310         params->nb_wc = 0;
 311
 312         const char *token = value;
 313
 314         while (isdigit(token[0])) {
 315                 char *rval;
 316                 unsigned int core = strtoul(token, &rval, 10);
 317
 318                 if (core >= RTE_MAX_LCORE) {
 319                         CS_LOG_ERR("Invalid worker core %u, should be smaller "
 320                                    "than %u.\n", core, RTE_MAX_LCORE);
 321                 }
 322                 params->wc_pool[params->nb_wc++] = (uint16_t)core;
 323                 token = (const char *)rval;
 324                 if (token[0] == '\0')
 325                         break;
 326                 token++;
 327         }
 328
 329         return 0;
 330 }
 331
 332 /** Parse name */
 333 static int
 334 parse_name_arg(const char *key __rte_unused,
 335                 const char *value, void *extra_args)
 336 {
 337         struct rte_cryptodev_pmd_init_params *params = extra_args;
 338
 339         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 340                 CS_LOG_ERR("Invalid name %s, should be less than "
 341                                 "%u bytes.\n", value,
 342                                 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 343                 return -EINVAL;
 344         }
 345
 346         strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
 347
 348         return 0;
 349 }
 350
 351 /** Parse slave */
 352 static int
 353 parse_slave_arg(const char *key __rte_unused,
 354                 const char *value, void *extra_args)
 355 {
 356         struct scheduler_init_params *param = extra_args;
 357
 358         if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
 359                 CS_LOG_ERR("Too many slaves.\n");
 360                 return -ENOMEM;
 361         }
 362
 363         strncpy(param->slave_names[param->nb_slaves++], value,
 364                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 365
 366         return 0;
 367 }
 368
 369 static int
 370 parse_mode_arg(const char *key __rte_unused,
 371                 const char *value, void *extra_args)
 372 {
 373         struct scheduler_init_params *param = extra_args;
 374         uint32_t i;
 375
 376         for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 377                 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
 378                         param->mode = (enum rte_cryptodev_scheduler_mode)
 379                                         scheduler_mode_map[i].val;
 380                         break;
 381                 }
 382         }
 383
 384         if (i == RTE_DIM(scheduler_mode_map)) {
 385                 CS_LOG_ERR("Unrecognized input.\n");
 386                 return -EINVAL;
 387         }
 388
 389         return 0;
 390 }
 391
 392 static int
 393 parse_ordering_arg(const char *key __rte_unused,
 394                 const char *value, void *extra_args)
 395 {
 396         struct scheduler_init_params *param = extra_args;
 397         uint32_t i;
 398
 399         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 400                 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
 401                         param->enable_ordering =
 402                                         scheduler_ordering_map[i].val;
 403                         break;
 404                 }
 405         }
 406
 407         if (i == RTE_DIM(scheduler_ordering_map)) {
 408                 CS_LOG_ERR("Unrecognized input.\n");
 409                 return -EINVAL;
 410         }
 411
 412         return 0;
 413 }
 414
 415 static int
 416 scheduler_parse_init_params(struct scheduler_init_params *params,
 417                 const char *input_args)
 418 {
 419         struct rte_kvargs *kvlist = NULL;
 420         int ret = 0;
 421
 422         if (params == NULL)
 423                 return -EINVAL;
 424
 425         if (input_args) {
 426                 kvlist = rte_kvargs_parse(input_args,
 427                                 scheduler_valid_params);
 428                 if (kvlist == NULL)
 429                         return -1;
 430
 431                 ret = rte_kvargs_process(kvlist,
 432                                 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
 433                                 &parse_integer_arg,
 434                                 &params->def_p.max_nb_queue_pairs);
 435                 if (ret < 0)
 436                         goto free_kvlist;
 437
 438                 ret = rte_kvargs_process(kvlist,
 439                                 RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
 440                                 &parse_integer_arg,
 441                                 &params->def_p.max_nb_sessions);
 442                 if (ret < 0)
 443                         goto free_kvlist;
 444
 445                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
 446                                 &parse_integer_arg,
 447                                 &params->def_p.socket_id);
 448                 if (ret < 0)
 449                         goto free_kvlist;
 450
 451                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
 452                                 &parse_coremask_arg,
 453                                 params);
 454                 if (ret < 0)
 455                         goto free_kvlist;
 456
 457                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
 458                                 &parse_corelist_arg,
 459                                 params);
 460                 if (ret < 0)
 461                         goto free_kvlist;
 462
 463                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
 464                                 &parse_name_arg,
 465                                 &params->def_p);
 466                 if (ret < 0)
 467                         goto free_kvlist;
 468
 469                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
 470                                 &parse_slave_arg, params);
 471                 if (ret < 0)
 472                         goto free_kvlist;
 473
 474                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
 475                                 &parse_mode_arg, params);
 476                 if (ret < 0)
 477                         goto free_kvlist;
 478
 479                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
 480                                 &parse_ordering_arg, params);
 481                 if (ret < 0)
 482                         goto free_kvlist;
 483         }
 484
 485 free_kvlist:
 486         rte_kvargs_free(kvlist);
 487         return ret;
 488 }
 489
 490 static int
 491 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
 492 {
 493         struct scheduler_init_params init_params = {
 494                 .def_p = {
 495                         "",
 496                         sizeof(struct scheduler_ctx),
 497                         rte_socket_id(),
 498                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 499                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
 500                 },
 501                 .nb_slaves = 0,
 502                 .mode = CDEV_SCHED_MODE_NOT_SET,
 503                 .enable_ordering = 0,
 504                 .slave_names = { {0} }
 505         };
 506         const char *name;
 507
 508         name = rte_vdev_device_name(vdev);
 509         if (name == NULL)
 510                 return -EINVAL;
 511
 512         scheduler_parse_init_params(&init_params,
 513                                     rte_vdev_device_args(vdev));
 514
 515
 516         return cryptodev_scheduler_create(name,
 517                                         vdev,
 518                                         &init_params);
 519 }
 520
 521 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
 522         .probe = cryptodev_scheduler_probe,
 523         .remove = cryptodev_scheduler_remove
 524 };
 525
 526 static struct cryptodev_driver scheduler_crypto_drv;
 527
 528 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
 529         cryptodev_scheduler_pmd_drv);
 530 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
 531         "max_nb_queue_pairs=<int> "
 532         "max_nb_sessions=<int> "
 533         "socket_id=<int> "
 534         "slave=<name>");
 535 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
 536                 cryptodev_scheduler_pmd_drv,
 537                 cryptodev_driver_id);