From 4d232ff5ac70f5a16fa4e324bb0dee073785b0eb Mon Sep 17 00:00:00 2001 From: Dastin Wilski Date: Fri, 26 Nov 2021 14:57:51 +0000 Subject: [PATCH] dpdk-cryptodev: scalable session count Originally cryptodev allocates mempools for seesion and session private data during its initialization. Moreover the size of these mempools are fixed resulting in limited session count (up to value specified in CRYPTODEV_NB_SESSION macro). This patch allows for session count to scale up by allocating new mempools as they are needed during session creation. Type: improvement Signed-off-by: Dastin Wilski Change-Id: I6ae240b474d3089d3ff50ca5bc7ff48f149983db --- src/plugins/dpdk/cryptodev/cryptodev.c | 230 +++++++++++++++++++-------------- src/plugins/dpdk/cryptodev/cryptodev.h | 9 +- 2 files changed, 143 insertions(+), 96 deletions(-) diff --git a/src/plugins/dpdk/cryptodev/cryptodev.c b/src/plugins/dpdk/cryptodev/cryptodev.c index d52fa407ec5..ab07e60519a 100644 --- a/src/plugins/dpdk/cryptodev/cryptodev.c +++ b/src/plugins/dpdk/cryptodev/cryptodev.c @@ -310,7 +310,7 @@ cryptodev_sess_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, if (cryptodev_check_supported_vnet_alg (key) == 0) return; - vec_validate (ckey->keys, idx); + vec_validate (ckey->keys, vec_len (cmt->per_numa_data) - 1); vec_foreach_index (i, ckey->keys) vec_validate (ckey->keys[i], CRYPTODEV_N_OP_TYPES - 1); } @@ -322,6 +322,50 @@ cryptodev_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop, cryptodev_sess_handler (vm, kop, idx, 8); } +clib_error_t * +allocate_session_pools (u32 numa_node, + cryptodev_session_pool_t *sess_pools_elt, u32 len) +{ + cryptodev_main_t *cmt = &cryptodev_main; + u8 *name; + clib_error_t *error = NULL; + + name = format (0, "vcryptodev_sess_pool_%u_%c", numa_node, len); + sess_pools_elt->sess_pool = rte_cryptodev_sym_session_pool_create ( + (char *) name, CRYPTODEV_NB_SESSION, 0, 0, 0, numa_node); + + if (!sess_pools_elt->sess_pool) + { + error = clib_error_return (0, "Not enough memory for mp %s", name); + goto clear_mempools; + } + vec_free (name); + + name = format (0, "cryptodev_sess_pool_%u_%c", numa_node, len); + sess_pools_elt->sess_priv_pool = rte_mempool_create ( + (char *) name, CRYPTODEV_NB_SESSION * (cmt->drivers_cnt), cmt->sess_sz, 0, + 0, NULL, NULL, NULL, NULL, numa_node, 0); + + if (!sess_pools_elt->sess_priv_pool) + { + error = clib_error_return (0, "Not enough memory for mp %s", name); + goto clear_mempools; + } + vec_free (name); + +clear_mempools: + if (error) + { + vec_free (name); + if (sess_pools_elt->sess_pool) + rte_mempool_free (sess_pools_elt->sess_pool); + if (sess_pools_elt->sess_priv_pool) + rte_mempool_free (sess_pools_elt->sess_priv_pool); + return error; + } + return 0; +} + int cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx, u32 aad_len) @@ -331,40 +375,69 @@ cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx, cryptodev_inst_t *dev_inst; vnet_crypto_key_t *key = vnet_crypto_get_key (idx); struct rte_mempool *sess_pool, *sess_priv_pool; + cryptodev_session_pool_t *sess_pools_elt; cryptodev_key_t *ckey = vec_elt_at_index (cmt->keys, idx); struct rte_crypto_sym_xform xforms_enc[2] = { { 0 } }; struct rte_crypto_sym_xform xforms_dec[2] = { { 0 } }; struct rte_cryptodev_sym_session *sessions[CRYPTODEV_N_OP_TYPES] = { 0 }; u32 numa_node = vm->numa_node; - int ret; + clib_error_t *error; + int ret = 0; + u8 found = 0; numa_data = vec_elt_at_index (cmt->per_numa_data, numa_node); - sess_pool = numa_data->sess_pool; - sess_priv_pool = numa_data->sess_priv_pool; - sessions[CRYPTODEV_OP_TYPE_ENCRYPT] = - rte_cryptodev_sym_session_create (sess_pool); - if (!sessions[CRYPTODEV_OP_TYPE_ENCRYPT]) + clib_spinlock_lock (&cmt->tlock); + vec_foreach (sess_pools_elt, numa_data->sess_pools) { - ret = -1; - goto clear_key; + if (sess_pools_elt->sess_pool == NULL) + { + error = allocate_session_pools (numa_node, sess_pools_elt, + vec_len (numa_data->sess_pools) - 1); + if (error) + { + ret = -1; + goto clear_key; + } + } + if (rte_mempool_avail_count (sess_pools_elt->sess_pool) >= 2) + { + found = 1; + break; + } } - sessions[CRYPTODEV_OP_TYPE_DECRYPT] = - rte_cryptodev_sym_session_create (sess_pool); - if (!sessions[CRYPTODEV_OP_TYPE_DECRYPT]) + if (found == 0) { - ret = -1; - goto clear_key; + vec_add2 (numa_data->sess_pools, sess_pools_elt, 1); + error = allocate_session_pools (numa_node, sess_pools_elt, + vec_len (numa_data->sess_pools) - 1); + if (error) + { + ret = -1; + goto clear_key; + } } + sess_pool = sess_pools_elt->sess_pool; + sess_priv_pool = sess_pools_elt->sess_priv_pool; + + sessions[CRYPTODEV_OP_TYPE_ENCRYPT] = + rte_cryptodev_sym_session_create (sess_pool); + + sessions[CRYPTODEV_OP_TYPE_DECRYPT] = + rte_cryptodev_sym_session_create (sess_pool); + if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) ret = prepare_linked_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key); else ret = prepare_aead_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key, aad_len); if (ret) - return 0; + { + ret = -1; + goto clear_key; + } if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) prepare_linked_xform (xforms_dec, CRYPTODEV_OP_TYPE_DECRYPT, key); @@ -390,7 +463,7 @@ cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx, dev_id, sessions[CRYPTODEV_OP_TYPE_DECRYPT], xforms_dec, sess_priv_pool); if (ret < 0) - return ret; + goto clear_key; } sessions[CRYPTODEV_OP_TYPE_ENCRYPT]->opaque_data = aad_len; @@ -408,6 +481,7 @@ clear_key: cryptodev_session_del (sessions[CRYPTODEV_OP_TYPE_ENCRYPT]); cryptodev_session_del (sessions[CRYPTODEV_OP_TYPE_DECRYPT]); } + clib_spinlock_unlock (&cmt->tlock); return ret; } @@ -1017,44 +1091,22 @@ cryptodev_probe (vlib_main_t *vm, u32 n_workers) } static void -cryptodev_get_max_sz (u32 *max_sess_sz, u32 *max_dp_sz) +is_drv_unique (u32 driver_id, u32 **unique_drivers) { - cryptodev_main_t *cmt = &cryptodev_main; - cryptodev_inst_t *cinst; - u32 max_sess = 0, max_dp = 0; + u32 *unique_elt; + u8 found = 0; - vec_foreach (cinst, cmt->cryptodev_inst) + vec_foreach (unique_elt, *unique_drivers) { - u32 sess_sz = rte_cryptodev_sym_get_private_session_size (cinst->dev_id); - u32 dp_sz = rte_cryptodev_get_raw_dp_ctx_size (cinst->dev_id); - - max_sess = clib_max (sess_sz, max_sess); - max_dp = clib_max (dp_sz, max_dp); + if (*unique_elt == driver_id) + { + found = 1; + break; + } } - *max_sess_sz = max_sess; - *max_dp_sz = max_dp; -} - -static void -dpdk_disable_cryptodev_engine (vlib_main_t *vm) -{ - vlib_thread_main_t *tm = vlib_get_thread_main (); - cryptodev_main_t *cmt = &cryptodev_main; - u32 i; - - for (i = (vlib_num_workers () > 0); i < tm->n_vlib_mains; i++) - { - u32 numa = vlib_get_main_by_index (i)->numa_node; - cryptodev_numa_data_t *numa_data; - - vec_validate (cmt->per_numa_data, numa); - numa_data = cmt->per_numa_data + numa; - if (numa_data->sess_pool) - rte_mempool_free (numa_data->sess_pool); - if (numa_data->sess_priv_pool) - rte_mempool_free (numa_data->sess_priv_pool); - } + if (!found) + vec_add1 (*unique_drivers, driver_id); } clib_error_t * @@ -1064,28 +1116,51 @@ dpdk_cryptodev_init (vlib_main_t * vm) vlib_thread_main_t *tm = vlib_get_thread_main (); cryptodev_engine_thread_t *cet; cryptodev_numa_data_t *numa_data; - struct rte_mempool *mp; + cryptodev_inst_t *dev_inst; + u32 node; + u8 nodes = 0; u32 skip_master = vlib_num_workers () > 0; u32 n_workers = tm->n_vlib_mains - skip_master; - u32 numa = vm->numa_node; - u32 sess_sz, dp_sz; u32 eidx; u32 i; - u8 *name = 0; + u32 *unique_drivers = 0; clib_error_t *error; cmt->iova_mode = rte_eal_iova_mode (); - vec_validate (cmt->per_numa_data, vm->numa_node); + clib_bitmap_foreach (node, tm->cpu_socket_bitmap) + { + if (node >= nodes) + nodes = node; + } + + vec_validate (cmt->per_numa_data, nodes); + vec_foreach (numa_data, cmt->per_numa_data) + { + vec_validate (numa_data->sess_pools, 0); + } /* probe all cryptodev devices and get queue info */ if (cryptodev_probe (vm, n_workers) < 0) { - error = clib_error_return (0, "Failed to configure cryptodev"); + error = clib_error_return (0, "Not enough cryptodev resources"); goto err_handling; } - cryptodev_get_max_sz (&sess_sz, &dp_sz); + vec_foreach (dev_inst, cmt->cryptodev_inst) + { + u32 dev_id = dev_inst->dev_id; + struct rte_cryptodev *cdev = rte_cryptodev_pmd_get_dev (dev_id); + u32 driver_id = cdev->driver_id; + is_drv_unique (driver_id, &unique_drivers); + + u32 sess_sz = + rte_cryptodev_sym_get_private_session_size (dev_inst->dev_id); + cmt->sess_sz = clib_max (cmt->sess_sz, sess_sz); + } + + cmt->drivers_cnt = vec_len (unique_drivers); + vec_free (unique_drivers); clib_bitmap_vec_validate (cmt->active_cdev_inst_mask, tm->n_vlib_mains); clib_spinlock_init (&cmt->tlock); @@ -1095,46 +1170,13 @@ dpdk_cryptodev_init (vlib_main_t * vm) for (i = skip_master; i < tm->n_vlib_mains; i++) { cet = cmt->per_thread_data + i; - numa = vlib_get_main_by_index (i)->numa_node; - vec_validate (cmt->per_numa_data, numa); - numa_data = vec_elt_at_index (cmt->per_numa_data, numa); - - if (!numa_data->sess_pool) + if (cryptodev_assign_resource (cet, 0, CRYPTODEV_RESOURCE_ASSIGN_AUTO) < + 0) { - /* create session pool for the numa node */ - name = format (0, "vcryptodev_sess_pool_%u%c", numa, 0); - mp = rte_cryptodev_sym_session_pool_create ( - (char *) name, CRYPTODEV_NB_SESSION, 0, 0, 0, numa); - if (!mp) - { - error = - clib_error_return (0, "Not enough memory for mp %s", name); - goto err_handling; - } - vec_free (name); - - numa_data->sess_pool = mp; - - /* create session private pool for the numa node */ - name = format (0, "cryptodev_sess_pool_%u%c", numa, 0); - mp = - rte_mempool_create ((char *) name, CRYPTODEV_NB_SESSION, sess_sz, - 0, 0, NULL, NULL, NULL, NULL, numa, 0); - if (!mp) - { - error = - clib_error_return (0, "Not enough memory for mp %s", name); - vec_free (name); - goto err_handling; - } - - vec_free (name); - - numa_data->sess_priv_pool = mp; + error = clib_error_return (0, "Failed to configure cryptodev"); + goto err_handling; } - - cryptodev_assign_resource (cet, 0, CRYPTODEV_RESOURCE_ASSIGN_AUTO); } /* register handler */ @@ -1160,7 +1202,5 @@ dpdk_cryptodev_init (vlib_main_t * vm) return 0; err_handling: - dpdk_disable_cryptodev_engine (vm); - return error; } diff --git a/src/plugins/dpdk/cryptodev/cryptodev.h b/src/plugins/dpdk/cryptodev/cryptodev.h index 3b47b43f538..80719ee69c8 100644 --- a/src/plugins/dpdk/cryptodev/cryptodev.h +++ b/src/plugins/dpdk/cryptodev/cryptodev.h @@ -27,7 +27,7 @@ #define CRYPTODEV_MAX_INFLIGHT (CRYPTODEV_NB_CRYPTO_OPS - 1) #define CRYPTODEV_AAD_MASK (CRYPTODEV_NB_CRYPTO_OPS - 1) #define CRYPTODEV_DEQ_CACHE_SZ 32 -#define CRYPTODEV_NB_SESSION 10240 +#define CRYPTODEV_NB_SESSION 4096 #define CRYPTODEV_MAX_IV_SIZE 16 #define CRYPTODEV_MAX_AAD_SIZE 16 #define CRYPTODEV_MAX_N_SGL 8 /**< maximum number of segments */ @@ -120,6 +120,11 @@ typedef struct { struct rte_mempool *sess_pool; struct rte_mempool *sess_priv_pool; +} cryptodev_session_pool_t; + +typedef struct +{ + cryptodev_session_pool_t *sess_pools; } cryptodev_numa_data_t; typedef struct @@ -170,6 +175,8 @@ typedef struct clib_bitmap_t *active_cdev_inst_mask; clib_spinlock_t tlock; cryptodev_capability_t *supported_caps; + u32 sess_sz; + u32 drivers_cnt; u8 is_raw_api; } cryptodev_main_t; -- 2.16.6