1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
13 #include <rte_malloc.h>
14 #include <rte_cycles.h>
15 #include <rte_vhost.h>
16 #include <rte_cryptodev.h>
17 #include <rte_vhost_crypto.h>
18 #include <rte_string_fns.h>
20 #include <cmdline_rdline.h>
21 #include <cmdline_parse.h>
22 #include <cmdline_parse_string.h>
25 #define NB_VIRTIO_QUEUES (1)
26 #define MAX_PKT_BURST (64)
27 #define MAX_IV_LEN (32)
28 #define NB_MEMPOOL_OBJS (8192)
29 #define NB_CRYPTO_DESCRIPTORS (4096)
30 #define NB_CACHE_OBJS (128)
31 #define SESSION_MAP_ENTRIES (1024)
32 #define REFRESH_TIME_SEC (3)
34 #define MAX_NB_SOCKETS (4)
35 #define MAX_NB_WORKER_CORES (16)
39 char *socket_files[MAX_NB_SOCKETS];
45 struct vhost_crypto_info {
46 int vids[MAX_NB_SOCKETS];
48 struct rte_mempool *sess_pool;
49 struct rte_mempool *cop_pool;
52 uint32_t nb_inflight_ops;
53 volatile uint32_t initialized[MAX_NB_SOCKETS];
54 } __rte_cache_aligned;
56 struct vhost_crypto_options {
57 struct lcore_option los[MAX_NB_WORKER_CORES];
58 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
61 uint32_t guest_polling;
64 #define CONFIG_KEYWORD "config"
65 #define SOCKET_FILE_KEYWORD "socket-file"
66 #define ZERO_COPY_KEYWORD "zero-copy"
67 #define POLLING_KEYWORD "guest-polling"
69 #define NB_SOCKET_FIELDS (2)
72 find_lo(uint32_t lcore_id)
76 for (i = 0; i < options.nb_los; i++)
77 if (options.los[i].lcore_id == lcore_id)
83 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
85 parse_socket_arg(char *arg)
89 char *str_fld[NB_SOCKET_FIELDS];
90 struct lcore_option *lo;
94 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
96 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
101 lcore_id = strtoul(str_fld[0], &end, 0);
102 if (errno != 0 || end == str_fld[0] || lcore_id > 255)
105 idx = find_lo(lcore_id);
106 if (idx == UINT32_MAX) {
107 if (options.nb_los == MAX_NB_WORKER_CORES)
109 lo = &options.los[options.nb_los];
110 lo->lcore_id = lcore_id;
113 lo = &options.los[idx];
115 nb_sockets = lo->nb_sockets;
117 if (nb_sockets >= MAX_NB_SOCKETS) {
118 RTE_LOG(ERR, USER1, "Too many socket files!\n");
122 lo->socket_files[nb_sockets] = strdup(str_fld[1]);
123 if (!lo->socket_files[nb_sockets]) {
124 RTE_LOG(ERR, USER1, "Insufficient memory\n");
134 parse_config(char *q_arg)
136 struct lcore_option *lo;
138 const char *p, *p0 = q_arg;
146 uint32_t flds[_NUM_FLD];
147 char *str_fld[_NUM_FLD];
151 while ((p = strchr(p0, '(')) != NULL) {
158 if (size >= sizeof(s))
161 snprintf(s, sizeof(s), "%.*s", size, p);
162 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
165 for (i = 0; i < _NUM_FLD; i++) {
167 flds[i] = strtoul(str_fld[i], &end, 0);
168 if (errno != 0 || end == str_fld[i] || flds[i] > 255)
172 if (flds[FLD_LCORE] > RTE_MAX_LCORE)
175 i = find_lo(flds[FLD_LCORE]);
176 if (i == UINT32_MAX) {
177 if (options.nb_los == MAX_NB_WORKER_CORES)
179 lo = &options.los[options.nb_los];
182 lo = &options.los[i];
184 lo->lcore_id = flds[FLD_LCORE];
185 lo->cid = flds[FLD_CID];
186 lo->qid = flds[FLD_QID];
193 vhost_crypto_usage(const char *prgname)
195 printf("%s [EAL options] --\n"
196 " --%s <lcore>,SOCKET-FILE-PATH\n"
197 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]"
199 " --%s: guest polling\n",
200 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD,
201 ZERO_COPY_KEYWORD, POLLING_KEYWORD);
205 vhost_crypto_parse_args(int argc, char **argv)
208 char *prgname = argv[0];
211 struct option lgopts[] = {
212 {SOCKET_FILE_KEYWORD, required_argument, 0, 0},
213 {CONFIG_KEYWORD, required_argument, 0, 0},
214 {ZERO_COPY_KEYWORD, no_argument, 0, 0},
215 {POLLING_KEYWORD, no_argument, 0, 0},
221 while ((opt = getopt_long(argc, argvopt, "s:",
222 lgopts, &option_index)) != EOF) {
226 if (strcmp(lgopts[option_index].name,
227 SOCKET_FILE_KEYWORD) == 0) {
228 ret = parse_socket_arg(optarg);
230 vhost_crypto_usage(prgname);
233 } else if (strcmp(lgopts[option_index].name,
234 CONFIG_KEYWORD) == 0) {
235 ret = parse_config(optarg);
237 vhost_crypto_usage(prgname);
240 } else if (strcmp(lgopts[option_index].name,
241 ZERO_COPY_KEYWORD) == 0) {
243 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
244 } else if (strcmp(lgopts[option_index].name,
245 POLLING_KEYWORD) == 0) {
246 options.guest_polling = 1;
248 vhost_crypto_usage(prgname);
263 struct vhost_crypto_info *info = NULL;
268 ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
270 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
274 for (i = 0; i < options.nb_los; i++) {
275 for (j = 0; j < options.los[i].nb_sockets; j++) {
276 if (strcmp(path, options.los[i].socket_files[j]) == 0) {
277 info = options.infos[i];
287 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
291 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
292 rte_lcore_to_socket_id(options.los[i].lcore_id));
294 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
298 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
300 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
301 options.zero_copy == 1 ? "enable" : "disable");
306 info->initialized[j] = 1;
310 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
316 destroy_device(int vid)
318 struct vhost_crypto_info *info = NULL;
321 for (i = 0; i < options.nb_los; i++) {
322 for (j = 0; j < options.los[i].nb_sockets; j++) {
323 if (options.infos[i]->vids[j] == vid) {
324 info = options.infos[i];
333 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
339 } while (info->nb_inflight_ops);
341 info->initialized[j] = 0;
345 rte_vhost_crypto_free(vid);
347 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
350 static const struct vhost_device_ops virtio_crypto_device_ops = {
351 .new_device = new_device,
352 .destroy_device = destroy_device,
355 __attribute__((unused))
356 static void clrscr(void)
358 system("@cls||clear");
362 vhost_crypto_worker(void *arg)
364 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
365 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
366 struct vhost_crypto_info *info = arg;
368 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
369 uint32_t lcore_id = rte_lcore_id();
370 uint32_t burst_size = MAX_PKT_BURST;
372 uint32_t to_fetch, fetched;
376 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
378 for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
379 if (rte_crypto_op_bulk_alloc(info->cop_pool,
380 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
381 burst_size) < burst_size) {
382 RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
389 for (i = 0; i < info->nb_vids; i++) {
390 if (unlikely(info->initialized[i] == 0))
393 for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
394 to_fetch = RTE_MIN(burst_size,
395 (NB_CRYPTO_DESCRIPTORS -
396 info->nb_inflight_ops));
397 fetched = rte_vhost_crypto_fetch_requests(
398 info->vids[i], j, ops[j],
400 info->nb_inflight_ops +=
401 rte_cryptodev_enqueue_burst(
402 info->cid, info->qid, ops[j],
404 if (unlikely(rte_crypto_op_bulk_alloc(
406 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
407 ops[j], fetched) < fetched)) {
408 RTE_LOG(ERR, USER1, "Failed realloc\n");
412 fetched = rte_cryptodev_dequeue_burst(
413 info->cid, info->qid,
414 ops_deq[j], RTE_MIN(burst_size,
415 info->nb_inflight_ops));
416 fetched = rte_vhost_crypto_finalize_requests(
417 ops_deq[j], fetched, callfds,
420 info->nb_inflight_ops -= fetched;
422 if (!options.guest_polling) {
423 for (k = 0; k < nb_callfds; k++)
424 eventfd_write(callfds[k],
428 rte_mempool_put_bulk(info->cop_pool,
429 (void **)ops_deq[j], fetched);
442 for (i = 0; i < options.nb_los; i++) {
443 struct lcore_option *lo = &options.los[i];
444 struct vhost_crypto_info *info = options.infos[i];
449 rte_mempool_free(info->cop_pool);
450 rte_mempool_free(info->sess_pool);
452 for (j = 0; j < lo->nb_sockets; j++) {
453 rte_vhost_driver_unregister(lo->socket_files[i]);
454 free(lo->socket_files[i]);
460 memset(&options, 0, sizeof(options));
464 main(int argc, char *argv[])
466 struct rte_cryptodev_qp_conf qp_conf = {NB_CRYPTO_DESCRIPTORS};
467 struct rte_cryptodev_config config;
468 struct rte_cryptodev_info dev_info;
470 uint32_t i, j, lcore;
473 ret = rte_eal_init(argc, argv);
479 ret = vhost_crypto_parse_args(argc, argv);
481 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
483 for (i = 0; i < options.nb_los; i++) {
484 struct lcore_option *lo = &options.los[i];
485 struct vhost_crypto_info *info;
487 info = rte_zmalloc_socket(NULL, sizeof(*info),
488 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
497 info->nb_vids = lo->nb_sockets;
499 rte_cryptodev_info_get(info->cid, &dev_info);
500 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
501 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb
502 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm
503 if (strstr(dev_info.driver_name,
504 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) ||
505 strstr(dev_info.driver_name,
506 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD)))
507 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
508 dev_info.driver_name);
513 if (dev_info.max_nb_queue_pairs < info->qid + 1) {
514 RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
515 dev_info.max_nb_queue_pairs);
519 config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
520 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
522 ret = rte_cryptodev_configure(info->cid, &config);
524 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
529 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
530 info->sess_pool = rte_mempool_create(name, SESSION_MAP_ENTRIES,
531 rte_cryptodev_sym_get_private_session_size(
532 info->cid), 64, 0, NULL, NULL, NULL, NULL,
533 rte_lcore_to_socket_id(lo->lcore_id), 0);
534 if (!info->sess_pool) {
535 RTE_LOG(ERR, USER1, "Failed to create mempool");
539 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
540 info->cop_pool = rte_crypto_op_pool_create(name,
541 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
543 rte_lcore_to_socket_id(lo->lcore_id));
545 if (!info->cop_pool) {
546 RTE_LOG(ERR, USER1, "Failed to create crypto pool");
551 options.infos[i] = info;
553 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
554 ret = rte_cryptodev_queue_pair_setup(info->cid, j,
555 &qp_conf, rte_lcore_to_socket_id(
559 RTE_LOG(ERR, USER1, "Failed to configure qp\n");
565 for (i = 0; i < options.nb_los; i++) {
566 struct lcore_option *lo = &options.los[i];
567 struct vhost_crypto_info *info = options.infos[i];
569 ret = rte_cryptodev_start(lo->cid);
571 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
575 if (rte_eal_remote_launch(vhost_crypto_worker, info,
577 RTE_LOG(ERR, USER1, "Failed to start worker lcore");
581 for (j = 0; j < lo->nb_sockets; j++) {
582 ret = rte_vhost_driver_register(lo->socket_files[j],
583 RTE_VHOST_USER_DEQUEUE_ZERO_COPY);
585 RTE_LOG(ERR, USER1, "socket %s already exists\n",
586 lo->socket_files[j]);
590 rte_vhost_driver_callback_register(lo->socket_files[j],
591 &virtio_crypto_device_ops);
593 ret = rte_vhost_driver_start(lo->socket_files[j]);
595 RTE_LOG(ERR, USER1, "failed to start vhost.\n");
601 RTE_LCORE_FOREACH(lcore)
602 rte_eal_wait_lcore(lcore);