drivers/crypto/scheduler/scheduler_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32 #include <rte_common.h>
  33 #include <rte_hexdump.h>
  34 #include <rte_cryptodev.h>
  35 #include <rte_cryptodev_pmd.h>
  36 #include <rte_bus_vdev.h>
  37 #include <rte_malloc.h>
  38 #include <rte_cpuflags.h>
  39 #include <rte_reorder.h>
  40
  41 #include "rte_cryptodev_scheduler.h"
  42 #include "scheduler_pmd_private.h"
  43
  44 uint8_t cryptodev_driver_id;
  45
  46 struct scheduler_init_params {
  47         struct rte_cryptodev_pmd_init_params def_p;
  48         uint32_t nb_slaves;
  49         enum rte_cryptodev_scheduler_mode mode;
  50         uint32_t enable_ordering;
  51         uint64_t wcmask;
  52         char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
  53                         [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
  54 };
  55
  56 #define RTE_CRYPTODEV_VDEV_NAME                 ("name")
  57 #define RTE_CRYPTODEV_VDEV_SLAVE                ("slave")
  58 #define RTE_CRYPTODEV_VDEV_MODE                 ("mode")
  59 #define RTE_CRYPTODEV_VDEV_ORDERING             ("ordering")
  60 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG        ("max_nb_queue_pairs")
  61 #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG      ("max_nb_sessions")
  62 #define RTE_CRYPTODEV_VDEV_SOCKET_ID            ("socket_id")
  63 #define RTE_CRYPTODEV_VDEV_COREMASK             ("coremask")
  64 #define RTE_CRYPTODEV_VDEV_CORELIST             ("corelist")
  65
  66 const char *scheduler_valid_params[] = {
  67         RTE_CRYPTODEV_VDEV_NAME,
  68         RTE_CRYPTODEV_VDEV_SLAVE,
  69         RTE_CRYPTODEV_VDEV_MODE,
  70         RTE_CRYPTODEV_VDEV_ORDERING,
  71         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
  72         RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
  73         RTE_CRYPTODEV_VDEV_SOCKET_ID,
  74         RTE_CRYPTODEV_VDEV_COREMASK,
  75         RTE_CRYPTODEV_VDEV_CORELIST
  76 };
  77
  78 struct scheduler_parse_map {
  79         const char *name;
  80         uint32_t val;
  81 };
  82
  83 const struct scheduler_parse_map scheduler_mode_map[] = {
  84         {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
  85                         CDEV_SCHED_MODE_ROUNDROBIN},
  86         {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
  87                         CDEV_SCHED_MODE_PKT_SIZE_DISTR},
  88         {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
  89                         CDEV_SCHED_MODE_FAILOVER},
  90         {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
  91                         CDEV_SCHED_MODE_MULTICORE}
  92 };
  93
  94 const struct scheduler_parse_map scheduler_ordering_map[] = {
  95                 {"enable", 1},
  96                 {"disable", 0}
  97 };
  98
  99 static int
 100 cryptodev_scheduler_create(const char *name,
 101                 struct rte_vdev_device *vdev,
 102                 struct scheduler_init_params *init_params)
 103 {
 104         struct rte_cryptodev *dev;
 105         struct scheduler_ctx *sched_ctx;
 106         uint32_t i;
 107         int ret;
 108
 109         dev = rte_cryptodev_pmd_create(name, &vdev->device,
 110                         &init_params->def_p);
 111         if (dev == NULL) {
 112                 CS_LOG_ERR("driver %s: failed to create cryptodev vdev",
 113                         name);
 114                 return -EFAULT;
 115         }
 116
 117         if (init_params->wcmask != 0)
 118                 RTE_LOG(INFO, PMD, "  workers core mask = %"PRIx64"\n",
 119                         init_params->wcmask);
 120
 121         dev->driver_id = cryptodev_driver_id;
 122         dev->dev_ops = rte_crypto_scheduler_pmd_ops;
 123
 124         sched_ctx = dev->data->dev_private;
 125         sched_ctx->max_nb_queue_pairs =
 126                         init_params->def_p.max_nb_queue_pairs;
 127
 128         if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
 129                 uint16_t i;
 130
 131                 sched_ctx->nb_wc = 0;
 132
 133                 for (i = 0; i < RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES; i++) {
 134                         if (init_params->wcmask & (1ULL << i)) {
 135                                 sched_ctx->wc_pool[sched_ctx->nb_wc++] = i;
 136                                 RTE_LOG(INFO, PMD,
 137                                         "  Worker core[%u]=%u added\n",
 138                                         sched_ctx->nb_wc-1, i);
 139                         }
 140                 }
 141         }
 142
 143         if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
 144                         init_params->mode < CDEV_SCHED_MODE_COUNT) {
 145                 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
 146                         init_params->mode);
 147                 if (ret < 0) {
 148                         rte_cryptodev_pmd_release_device(dev);
 149                         return ret;
 150                 }
 151
 152                 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 153                         if (scheduler_mode_map[i].val != sched_ctx->mode)
 154                                 continue;
 155
 156                         RTE_LOG(INFO, PMD, "  Scheduling mode = %s\n",
 157                                         scheduler_mode_map[i].name);
 158                         break;
 159                 }
 160         }
 161
 162         sched_ctx->reordering_enabled = init_params->enable_ordering;
 163
 164         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 165                 if (scheduler_ordering_map[i].val !=
 166                                 sched_ctx->reordering_enabled)
 167                         continue;
 168
 169                 RTE_LOG(INFO, PMD, "  Packet ordering = %s\n",
 170                                 scheduler_ordering_map[i].name);
 171
 172                 break;
 173         }
 174
 175         for (i = 0; i < init_params->nb_slaves; i++) {
 176                 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
 177                         rte_zmalloc_socket(
 178                                 NULL,
 179                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
 180                                 SOCKET_ID_ANY);
 181
 182                 if (!sched_ctx->init_slave_names[
 183                                 sched_ctx->nb_init_slaves]) {
 184                         CS_LOG_ERR("driver %s: Insufficient memory",
 185                                         name);
 186                         return -ENOMEM;
 187                 }
 188
 189                 strncpy(sched_ctx->init_slave_names[
 190                                         sched_ctx->nb_init_slaves],
 191                                 init_params->slave_names[i],
 192                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 193
 194                 sched_ctx->nb_init_slaves++;
 195         }
 196
 197         /*
 198          * Initialize capabilities structure as an empty structure,
 199          * in case device information is requested when no slaves are attached
 200          */
 201         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 202                         sizeof(struct rte_cryptodev_capabilities),
 203                         0, SOCKET_ID_ANY);
 204
 205         if (!sched_ctx->capabilities) {
 206                 RTE_LOG(ERR, PMD, "Not enough memory for capability "
 207                                 "information\n");
 208                 return -ENOMEM;
 209         }
 210
 211         return 0;
 212 }
 213
 214 static int
 215 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
 216 {
 217         const char *name;
 218         struct rte_cryptodev *dev;
 219         struct scheduler_ctx *sched_ctx;
 220
 221         if (vdev == NULL)
 222                 return -EINVAL;
 223
 224         name = rte_vdev_device_name(vdev);
 225         dev = rte_cryptodev_pmd_get_named_dev(name);
 226         if (dev == NULL)
 227                 return -EINVAL;
 228
 229         sched_ctx = dev->data->dev_private;
 230
 231         if (sched_ctx->nb_slaves) {
 232                 uint32_t i;
 233
 234                 for (i = 0; i < sched_ctx->nb_slaves; i++)
 235                         rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
 236                                         sched_ctx->slaves[i].dev_id);
 237         }
 238
 239         return rte_cryptodev_pmd_destroy(dev);
 240 }
 241
 242 /** Parse integer from integer argument */
 243 static int
 244 parse_integer_arg(const char *key __rte_unused,
 245                 const char *value, void *extra_args)
 246 {
 247         int *i = (int *) extra_args;
 248
 249         *i = atoi(value);
 250         if (*i < 0) {
 251                 CS_LOG_ERR("Argument has to be positive.\n");
 252                 return -EINVAL;
 253         }
 254
 255         return 0;
 256 }
 257
 258 /** Parse integer from hexadecimal integer argument */
 259 static int
 260 parse_coremask_arg(const char *key __rte_unused,
 261                 const char *value, void *extra_args)
 262 {
 263         struct scheduler_init_params *params = extra_args;
 264
 265         params->wcmask = strtoull(value, NULL, 16);
 266
 267         return 0;
 268 }
 269
 270 /** Parse integer from list of integers argument */
 271 static int
 272 parse_corelist_arg(const char *key __rte_unused,
 273                 const char *value, void *extra_args)
 274 {
 275         struct scheduler_init_params *params = extra_args;
 276
 277         params->wcmask = 0ULL;
 278
 279         const char *token = value;
 280
 281         while (isdigit(token[0])) {
 282                 char *rval;
 283                 unsigned int core = strtoul(token, &rval, 10);
 284
 285                 params->wcmask |= 1ULL << core;
 286                 token = (const char *)rval;
 287                 if (token[0] == '\0')
 288                         break;
 289                 token++;
 290         }
 291
 292         return 0;
 293 }
 294
 295 /** Parse name */
 296 static int
 297 parse_name_arg(const char *key __rte_unused,
 298                 const char *value, void *extra_args)
 299 {
 300         struct rte_cryptodev_pmd_init_params *params = extra_args;
 301
 302         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 303                 CS_LOG_ERR("Invalid name %s, should be less than "
 304                                 "%u bytes.\n", value,
 305                                 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 306                 return -EINVAL;
 307         }
 308
 309         strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
 310
 311         return 0;
 312 }
 313
 314 /** Parse slave */
 315 static int
 316 parse_slave_arg(const char *key __rte_unused,
 317                 const char *value, void *extra_args)
 318 {
 319         struct scheduler_init_params *param = extra_args;
 320
 321         if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
 322                 CS_LOG_ERR("Too many slaves.\n");
 323                 return -ENOMEM;
 324         }
 325
 326         strncpy(param->slave_names[param->nb_slaves++], value,
 327                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 328
 329         return 0;
 330 }
 331
 332 static int
 333 parse_mode_arg(const char *key __rte_unused,
 334                 const char *value, void *extra_args)
 335 {
 336         struct scheduler_init_params *param = extra_args;
 337         uint32_t i;
 338
 339         for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 340                 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
 341                         param->mode = (enum rte_cryptodev_scheduler_mode)
 342                                         scheduler_mode_map[i].val;
 343                         break;
 344                 }
 345         }
 346
 347         if (i == RTE_DIM(scheduler_mode_map)) {
 348                 CS_LOG_ERR("Unrecognized input.\n");
 349                 return -EINVAL;
 350         }
 351
 352         return 0;
 353 }
 354
 355 static int
 356 parse_ordering_arg(const char *key __rte_unused,
 357                 const char *value, void *extra_args)
 358 {
 359         struct scheduler_init_params *param = extra_args;
 360         uint32_t i;
 361
 362         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 363                 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
 364                         param->enable_ordering =
 365                                         scheduler_ordering_map[i].val;
 366                         break;
 367                 }
 368         }
 369
 370         if (i == RTE_DIM(scheduler_ordering_map)) {
 371                 CS_LOG_ERR("Unrecognized input.\n");
 372                 return -EINVAL;
 373         }
 374
 375         return 0;
 376 }
 377
 378 static int
 379 scheduler_parse_init_params(struct scheduler_init_params *params,
 380                 const char *input_args)
 381 {
 382         struct rte_kvargs *kvlist = NULL;
 383         int ret = 0;
 384
 385         if (params == NULL)
 386                 return -EINVAL;
 387
 388         if (input_args) {
 389                 kvlist = rte_kvargs_parse(input_args,
 390                                 scheduler_valid_params);
 391                 if (kvlist == NULL)
 392                         return -1;
 393
 394                 ret = rte_kvargs_process(kvlist,
 395                                 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
 396                                 &parse_integer_arg,
 397                                 &params->def_p.max_nb_queue_pairs);
 398                 if (ret < 0)
 399                         goto free_kvlist;
 400
 401                 ret = rte_kvargs_process(kvlist,
 402                                 RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
 403                                 &parse_integer_arg,
 404                                 &params->def_p.max_nb_sessions);
 405                 if (ret < 0)
 406                         goto free_kvlist;
 407
 408                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
 409                                 &parse_integer_arg,
 410                                 &params->def_p.socket_id);
 411                 if (ret < 0)
 412                         goto free_kvlist;
 413
 414                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
 415                                 &parse_coremask_arg,
 416                                 params);
 417                 if (ret < 0)
 418                         goto free_kvlist;
 419
 420                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
 421                                 &parse_corelist_arg,
 422                                 params);
 423                 if (ret < 0)
 424                         goto free_kvlist;
 425
 426                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
 427                                 &parse_name_arg,
 428                                 &params->def_p);
 429                 if (ret < 0)
 430                         goto free_kvlist;
 431
 432                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
 433                                 &parse_slave_arg, params);
 434                 if (ret < 0)
 435                         goto free_kvlist;
 436
 437                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
 438                                 &parse_mode_arg, params);
 439                 if (ret < 0)
 440                         goto free_kvlist;
 441
 442                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
 443                                 &parse_ordering_arg, params);
 444                 if (ret < 0)
 445                         goto free_kvlist;
 446         }
 447
 448 free_kvlist:
 449         rte_kvargs_free(kvlist);
 450         return ret;
 451 }
 452
 453 static int
 454 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
 455 {
 456         struct scheduler_init_params init_params = {
 457                 .def_p = {
 458                         "",
 459                         sizeof(struct scheduler_ctx),
 460                         rte_socket_id(),
 461                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 462                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
 463                 },
 464                 .nb_slaves = 0,
 465                 .mode = CDEV_SCHED_MODE_NOT_SET,
 466                 .enable_ordering = 0,
 467                 .slave_names = { {0} }
 468         };
 469         const char *name;
 470
 471         name = rte_vdev_device_name(vdev);
 472         if (name == NULL)
 473                 return -EINVAL;
 474
 475         scheduler_parse_init_params(&init_params,
 476                                     rte_vdev_device_args(vdev));
 477
 478
 479         return cryptodev_scheduler_create(name,
 480                                         vdev,
 481                                         &init_params);
 482 }
 483
 484 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
 485         .probe = cryptodev_scheduler_probe,
 486         .remove = cryptodev_scheduler_remove
 487 };
 488
 489 static struct cryptodev_driver scheduler_crypto_drv;
 490
 491 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
 492         cryptodev_scheduler_pmd_drv);
 493 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
 494         "max_nb_queue_pairs=<int> "
 495         "max_nb_sessions=<int> "
 496         "socket_id=<int> "
 497         "slave=<name>");
 498 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
 499                 cryptodev_scheduler_pmd_drv,
 500                 cryptodev_driver_id);