examples/vhost_crypto/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <unistd.h>
   8 #include <stdbool.h>
   9 #include <assert.h>
  10 #include <getopt.h>
  11
  12 #include <rte_malloc.h>
  13 #include <rte_cycles.h>
  14 #include <rte_vhost.h>
  15 #include <rte_cryptodev.h>
  16 #include <rte_vhost_crypto.h>
  17 #include <rte_string_fns.h>
  18
  19 #include <cmdline_rdline.h>
  20 #include <cmdline_parse.h>
  21 #include <cmdline_parse_string.h>
  22 #include <cmdline.h>
  23
  24 #define NB_VIRTIO_QUEUES                (1)
  25 #define MAX_PKT_BURST                   (64)
  26 #define MAX_IV_LEN                      (32)
  27 #define NB_MEMPOOL_OBJS                 (8192)
  28 #define NB_CRYPTO_DESCRIPTORS           (4096)
  29 #define NB_CACHE_OBJS                   (128)
  30 #define SESSION_MAP_ENTRIES             (1024)
  31 #define REFRESH_TIME_SEC                (3)
  32
  33 #define MAX_NB_SOCKETS                  (4)
  34 #define MAX_NB_WORKER_CORES             (16)
  35
  36 struct lcore_option {
  37         uint32_t lcore_id;
  38         char *socket_files[MAX_NB_SOCKETS];
  39         uint32_t nb_sockets;
  40         uint8_t cid;
  41         uint16_t qid;
  42 };
  43
  44 struct vhost_crypto_info {
  45         int vids[MAX_NB_SOCKETS];
  46         uint32_t nb_vids;
  47         struct rte_mempool *sess_pool;
  48         struct rte_mempool *cop_pool;
  49         uint8_t cid;
  50         uint32_t qid;
  51         uint32_t nb_inflight_ops;
  52         volatile uint32_t initialized[MAX_NB_SOCKETS];
  53 } __rte_cache_aligned;
  54
  55 struct vhost_crypto_options {
  56         struct lcore_option los[MAX_NB_WORKER_CORES];
  57         struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
  58         uint32_t nb_los;
  59         uint32_t zero_copy;
  60         uint32_t guest_polling;
  61 } options;
  62
  63 #define CONFIG_KEYWORD          "config"
  64 #define SOCKET_FILE_KEYWORD     "socket-file"
  65 #define ZERO_COPY_KEYWORD       "zero-copy"
  66 #define POLLING_KEYWORD         "guest-polling"
  67
  68 #define NB_SOCKET_FIELDS        (2)
  69
  70 static uint32_t
  71 find_lo(uint32_t lcore_id)
  72 {
  73         uint32_t i;
  74
  75         for (i = 0; i < options.nb_los; i++)
  76                 if (options.los[i].lcore_id == lcore_id)
  77                         return i;
  78
  79         return UINT32_MAX;
  80 }
  81
  82 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
  83 static int
  84 parse_socket_arg(char *arg)
  85 {
  86         uint32_t nb_sockets;
  87         uint32_t lcore_id;
  88         char *str_fld[NB_SOCKET_FIELDS];
  89         struct lcore_option *lo;
  90         uint32_t idx;
  91         char *end;
  92
  93         if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
  94                                 NB_SOCKET_FIELDS) {
  95                 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
  96                 return -EINVAL;
  97         }
  98
  99         errno = 0;
 100         lcore_id = strtoul(str_fld[0], &end, 0);
 101         if (errno != 0 || end == str_fld[0] || lcore_id > 255)
 102                 return -EINVAL;
 103
 104         idx = find_lo(lcore_id);
 105         if (idx == UINT32_MAX) {
 106                 if (options.nb_los == MAX_NB_WORKER_CORES)
 107                         return -ENOMEM;
 108                 lo = &options.los[options.nb_los];
 109                 lo->lcore_id = lcore_id;
 110                 options.nb_los++;
 111         } else
 112                 lo = &options.los[idx];
 113
 114         nb_sockets = lo->nb_sockets;
 115
 116         if (nb_sockets >= MAX_NB_SOCKETS) {
 117                 RTE_LOG(ERR, USER1, "Too many socket files!\n");
 118                 return -ENOMEM;
 119         }
 120
 121         lo->socket_files[nb_sockets] = strdup(str_fld[1]);
 122         if (!lo->socket_files[nb_sockets]) {
 123                 RTE_LOG(ERR, USER1, "Insufficient memory\n");
 124                 return -ENOMEM;
 125         }
 126
 127         lo->nb_sockets++;
 128
 129         return 0;
 130 }
 131
 132 static int
 133 parse_config(char *q_arg)
 134 {
 135         struct lcore_option *lo;
 136         char s[256];
 137         const char *p, *p0 = q_arg;
 138         char *end;
 139         enum fieldnames {
 140                 FLD_LCORE = 0,
 141                 FLD_CID,
 142                 FLD_QID,
 143                 _NUM_FLD
 144         };
 145         uint32_t flds[_NUM_FLD];
 146         char *str_fld[_NUM_FLD];
 147         uint32_t i;
 148         uint32_t size;
 149
 150         while ((p = strchr(p0, '(')) != NULL) {
 151                 ++p;
 152                 p0 = strchr(p, ')');
 153                 if (p0 == NULL)
 154                         return -1;
 155
 156                 size = p0 - p;
 157                 if (size >= sizeof(s))
 158                         return -1;
 159
 160                 snprintf(s, sizeof(s), "%.*s", size, p);
 161                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
 162                                 _NUM_FLD)
 163                         return -1;
 164                 for (i = 0; i < _NUM_FLD; i++) {
 165                         errno = 0;
 166                         flds[i] = strtoul(str_fld[i], &end, 0);
 167                         if (errno != 0 || end == str_fld[i] || flds[i] > 255)
 168                                 return -EINVAL;
 169                 }
 170
 171                 if (flds[FLD_LCORE] > RTE_MAX_LCORE)
 172                         return -EINVAL;
 173
 174                 i = find_lo(flds[FLD_LCORE]);
 175                 if (i == UINT32_MAX) {
 176                         if (options.nb_los == MAX_NB_WORKER_CORES)
 177                                 return -ENOMEM;
 178                         lo = &options.los[options.nb_los];
 179                         options.nb_los++;
 180                 } else
 181                         lo = &options.los[i];
 182
 183                 lo->lcore_id = flds[FLD_LCORE];
 184                 lo->cid = flds[FLD_CID];
 185                 lo->qid = flds[FLD_QID];
 186         }
 187
 188         return 0;
 189 }
 190
 191 static void
 192 vhost_crypto_usage(const char *prgname)
 193 {
 194         printf("%s [EAL options] --\n"
 195                 "  --%s <lcore>,SOCKET-FILE-PATH\n"
 196                 "  --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]"
 197                 "  --%s: zero copy\n"
 198                 "  --%s: guest polling\n",
 199                 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD,
 200                 ZERO_COPY_KEYWORD, POLLING_KEYWORD);
 201 }
 202
 203 static int
 204 vhost_crypto_parse_args(int argc, char **argv)
 205 {
 206         int opt, ret;
 207         char *prgname = argv[0];
 208         char **argvopt;
 209         int option_index;
 210         struct option lgopts[] = {
 211                         {SOCKET_FILE_KEYWORD, required_argument, 0, 0},
 212                         {CONFIG_KEYWORD, required_argument, 0, 0},
 213                         {ZERO_COPY_KEYWORD, no_argument, 0, 0},
 214                         {POLLING_KEYWORD, no_argument, 0, 0},
 215                         {NULL, 0, 0, 0}
 216         };
 217
 218         argvopt = argv;
 219
 220         while ((opt = getopt_long(argc, argvopt, "s:",
 221                                   lgopts, &option_index)) != EOF) {
 222
 223                 switch (opt) {
 224                 case 0:
 225                         if (strcmp(lgopts[option_index].name,
 226                                         SOCKET_FILE_KEYWORD) == 0) {
 227                                 ret = parse_socket_arg(optarg);
 228                                 if (ret < 0) {
 229                                         vhost_crypto_usage(prgname);
 230                                         return ret;
 231                                 }
 232                         } else if (strcmp(lgopts[option_index].name,
 233                                         CONFIG_KEYWORD) == 0) {
 234                                 ret = parse_config(optarg);
 235                                 if (ret < 0) {
 236                                         vhost_crypto_usage(prgname);
 237                                         return ret;
 238                                 }
 239                         } else if (strcmp(lgopts[option_index].name,
 240                                         ZERO_COPY_KEYWORD) == 0) {
 241                                 options.zero_copy =
 242                                         RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
 243                         } else if (strcmp(lgopts[option_index].name,
 244                                         POLLING_KEYWORD) == 0) {
 245                                 options.guest_polling = 1;
 246                         } else {
 247                                 vhost_crypto_usage(prgname);
 248                                 return -EINVAL;
 249                         }
 250                         break;
 251                 default:
 252                         return -1;
 253                 }
 254         }
 255
 256         return 0;
 257 }
 258
 259 static int
 260 new_device(int vid)
 261 {
 262         struct vhost_crypto_info *info = NULL;
 263         char path[PATH_MAX];
 264         uint32_t i, j;
 265         int ret;
 266
 267         ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
 268         if (ret) {
 269                 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
 270                 return ret;
 271         }
 272
 273         for (i = 0; i < options.nb_los; i++) {
 274                 for (j = 0; j < options.los[i].nb_sockets; j++) {
 275                         if (strcmp(path, options.los[i].socket_files[j]) == 0) {
 276                                 info = options.infos[i];
 277                                 break;
 278                         }
 279                 }
 280
 281                 if (info)
 282                         break;
 283         }
 284
 285         if (!info) {
 286                 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
 287                 return -ENOENT;
 288         }
 289
 290         ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
 291                         rte_lcore_to_socket_id(options.los[i].lcore_id));
 292         if (ret) {
 293                 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
 294                 return ret;
 295         }
 296
 297         ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
 298         if (ret) {
 299                 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
 300                                 options.zero_copy == 1 ? "enable" : "disable");
 301                 return ret;
 302         }
 303
 304         info->vids[j] = vid;
 305         info->initialized[j] = 1;
 306
 307         rte_wmb();
 308
 309         RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
 310                         vid);
 311         return 0;
 312 }
 313
 314 static void
 315 destroy_device(int vid)
 316 {
 317         struct vhost_crypto_info *info = NULL;
 318         uint32_t i, j;
 319
 320         for (i = 0; i < options.nb_los; i++) {
 321                 for (j = 0; j < options.los[i].nb_sockets; j++) {
 322                         if (options.infos[i]->vids[j] == vid) {
 323                                 info = options.infos[i];
 324                                 break;
 325                         }
 326                 }
 327                 if (info)
 328                         break;
 329         }
 330
 331         if (!info) {
 332                 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
 333                 return;
 334         }
 335
 336         do {
 337
 338         } while (info->nb_inflight_ops);
 339
 340         info->initialized[j] = 0;
 341
 342         rte_wmb();
 343
 344         rte_vhost_crypto_free(vid);
 345
 346         RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
 347 }
 348
 349 static const struct vhost_device_ops virtio_crypto_device_ops = {
 350         .new_device =  new_device,
 351         .destroy_device = destroy_device,
 352 };
 353
 354 __attribute__((unused))
 355 static void clrscr(void)
 356 {
 357         system("@cls||clear");
 358 }
 359
 360 static int
 361 vhost_crypto_worker(void *arg)
 362 {
 363         struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
 364         struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
 365         struct vhost_crypto_info *info = arg;
 366         uint16_t nb_callfds;
 367         int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
 368         uint32_t lcore_id = rte_lcore_id();
 369         uint32_t burst_size = MAX_PKT_BURST;
 370         uint32_t i, j, k;
 371         uint32_t to_fetch, fetched;
 372
 373         int ret = 0;
 374
 375         RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
 376
 377         for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
 378                 if (rte_crypto_op_bulk_alloc(info->cop_pool,
 379                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
 380                                 burst_size) < burst_size) {
 381                         RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
 382                         ret = -1;
 383                         goto exit;
 384                 }
 385         }
 386
 387         while (1) {
 388                 for (i = 0; i < info->nb_vids; i++) {
 389                         if (unlikely(info->initialized[i] == 0))
 390                                 continue;
 391
 392                         for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
 393                                 to_fetch = RTE_MIN(burst_size,
 394                                                 (NB_CRYPTO_DESCRIPTORS -
 395                                                 info->nb_inflight_ops));
 396                                 fetched = rte_vhost_crypto_fetch_requests(
 397                                                 info->vids[i], j, ops[j],
 398                                                 to_fetch);
 399                                 info->nb_inflight_ops +=
 400                                                 rte_cryptodev_enqueue_burst(
 401                                                 info->cid, info->qid, ops[j],
 402                                                 fetched);
 403                                 if (unlikely(rte_crypto_op_bulk_alloc(
 404                                                 info->cop_pool,
 405                                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 406                                                 ops[j], fetched) < fetched)) {
 407                                         RTE_LOG(ERR, USER1, "Failed realloc\n");
 408                                         return -1;
 409                                 }
 410
 411                                 fetched = rte_cryptodev_dequeue_burst(
 412                                                 info->cid, info->qid,
 413                                                 ops_deq[j], RTE_MIN(burst_size,
 414                                                 info->nb_inflight_ops));
 415                                 fetched = rte_vhost_crypto_finalize_requests(
 416                                                 ops_deq[j], fetched, callfds,
 417                                                 &nb_callfds);
 418
 419                                 info->nb_inflight_ops -= fetched;
 420
 421                                 if (!options.guest_polling) {
 422                                         for (k = 0; k < nb_callfds; k++)
 423                                                 eventfd_write(callfds[k],
 424                                                                 (eventfd_t)1);
 425                                 }
 426
 427                                 rte_mempool_put_bulk(info->cop_pool,
 428                                                 (void **)ops_deq[j], fetched);
 429                         }
 430                 }
 431         }
 432 exit:
 433         return ret;
 434 }
 435
 436 static void
 437 free_resource(void)
 438 {
 439         uint32_t i, j;
 440
 441         for (i = 0; i < options.nb_los; i++) {
 442                 struct lcore_option *lo = &options.los[i];
 443                 struct vhost_crypto_info *info = options.infos[i];
 444
 445                 rte_mempool_free(info->cop_pool);
 446                 rte_mempool_free(info->sess_pool);
 447
 448                 for (j = 0; j < lo->nb_sockets; j++) {
 449                         rte_vhost_driver_unregister(lo->socket_files[i]);
 450                         free(lo->socket_files[i]);
 451                 }
 452
 453                 rte_free(info);
 454         }
 455
 456         memset(&options, 0, sizeof(options));
 457 }
 458
 459 int
 460 main(int argc, char *argv[])
 461 {
 462         struct rte_cryptodev_qp_conf qp_conf = {NB_CRYPTO_DESCRIPTORS};
 463         struct rte_cryptodev_config config;
 464         struct rte_cryptodev_info dev_info;
 465         char name[128];
 466         uint32_t i, j, lcore;
 467         int ret;
 468
 469         ret = rte_eal_init(argc, argv);
 470         if (ret < 0)
 471                 return -1;
 472         argc -= ret;
 473         argv += ret;
 474
 475         ret = vhost_crypto_parse_args(argc, argv);
 476         if (ret < 0)
 477                 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
 478
 479         for (i = 0; i < options.nb_los; i++) {
 480                 struct lcore_option *lo = &options.los[i];
 481                 struct vhost_crypto_info *info;
 482
 483                 info = rte_zmalloc_socket(NULL, sizeof(*info),
 484                                 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
 485                                                 lo->lcore_id));
 486                 if (!info) {
 487                         ret = -ENOMEM;
 488                         goto error_exit;
 489                 }
 490
 491                 info->cid = lo->cid;
 492                 info->qid = lo->qid;
 493                 info->nb_vids = lo->nb_sockets;
 494
 495                 rte_cryptodev_info_get(info->cid, &dev_info);
 496                 if (dev_info.max_nb_queue_pairs < info->qid + 1) {
 497                         RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
 498                                         dev_info.max_nb_queue_pairs);
 499                         goto error_exit;
 500                 }
 501
 502                 config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
 503                 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
 504
 505                 ret = rte_cryptodev_configure(info->cid, &config);
 506                 if (ret < 0) {
 507                         RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
 508                                         info->cid);
 509                         goto error_exit;
 510                 }
 511
 512                 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
 513                 info->sess_pool = rte_mempool_create(name, SESSION_MAP_ENTRIES,
 514                                 rte_cryptodev_sym_get_private_session_size(
 515                                 info->cid), 64, 0, NULL, NULL, NULL, NULL,
 516                                 rte_lcore_to_socket_id(lo->lcore_id), 0);
 517                 if (!info->sess_pool) {
 518                         RTE_LOG(ERR, USER1, "Failed to create mempool");
 519                         goto error_exit;
 520                 }
 521
 522                 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
 523                 info->cop_pool = rte_crypto_op_pool_create(name,
 524                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
 525                                 NB_CACHE_OBJS, 0,
 526                                 rte_lcore_to_socket_id(lo->lcore_id));
 527
 528                 if (!info->cop_pool) {
 529                         RTE_LOG(ERR, USER1, "Failed to create crypto pool");
 530                         ret = -ENOMEM;
 531                         goto error_exit;
 532                 }
 533
 534                 options.infos[i] = info;
 535
 536                 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
 537                         ret = rte_cryptodev_queue_pair_setup(info->cid, j,
 538                                         &qp_conf, rte_lcore_to_socket_id(
 539                                                         lo->lcore_id),
 540                                         info->sess_pool);
 541                         if (ret < 0) {
 542                                 RTE_LOG(ERR, USER1, "Failed to configure qp\n");
 543                                 goto error_exit;
 544                         }
 545                 }
 546         }
 547
 548         for (i = 0; i < options.nb_los; i++) {
 549                 struct lcore_option *lo = &options.los[i];
 550                 struct vhost_crypto_info *info = options.infos[i];
 551
 552                 ret = rte_cryptodev_start(lo->cid);
 553                 if (ret < 0) {
 554                         RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
 555                         goto error_exit;
 556                 }
 557
 558                 if (rte_eal_remote_launch(vhost_crypto_worker, info,
 559                                 lo->lcore_id) < 0) {
 560                         RTE_LOG(ERR, USER1, "Failed to start worker lcore");
 561                         goto error_exit;
 562                 }
 563
 564                 for (j = 0; j < lo->nb_sockets; j++) {
 565                         ret = rte_vhost_driver_register(lo->socket_files[j],
 566                                 RTE_VHOST_USER_DEQUEUE_ZERO_COPY);
 567                         if (ret < 0) {
 568                                 RTE_LOG(ERR, USER1, "socket %s already exists\n",
 569                                         lo->socket_files[j]);
 570                                 goto error_exit;
 571                         }
 572
 573                         rte_vhost_driver_callback_register(lo->socket_files[j],
 574                                 &virtio_crypto_device_ops);
 575
 576                         ret = rte_vhost_driver_start(lo->socket_files[j]);
 577                         if (ret < 0)  {
 578                                 RTE_LOG(ERR, USER1, "failed to start vhost.\n");
 579                                 goto error_exit;
 580                         }
 581                 }
 582         }
 583
 584         RTE_LCORE_FOREACH(lcore)
 585                 rte_eal_wait_lcore(lcore);
 586
 587         free_resource();
 588
 589         return 0;
 590
 591 error_exit:
 592
 593         free_resource();
 594
 595         return -1;
 596 }