#include <vlib/vlib.h>
#include <vnet/plugin/plugin.h>
#include <vnet/crypto/crypto.h>
-#include <vnet/vnet.h>
+#include <vnet/ipsec/ipsec.h>
#include <vpp/app/version.h>
#include <dpdk/buffer.h>
typedef struct
{
- struct rte_cryptodev_sym_session *keys[CRYPTODEV_N_OP_TYPES];
+ struct rte_cryptodev_sym_session ***keys;
} cryptodev_key_t;
typedef struct
typedef struct
{
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
struct rte_mempool *cop_pool;
struct rte_mempool *sess_pool;
struct rte_mempool *sess_priv_pool;
cryptodev_main_t cryptodev_main;
-static int
+static_always_inline int
prepare_aead_xform (struct rte_crypto_sym_xform *xform,
- cryptodev_op_type_t op_type,
- const vnet_crypto_key_t * key, u32 aad_len)
+ cryptodev_op_type_t op_type, const vnet_crypto_key_t *key,
+ u32 aad_len)
{
struct rte_crypto_aead_xform *aead_xform = &xform->aead;
memset (xform, 0, sizeof (*xform));
return 0;
}
-static int
+static_always_inline int
prepare_linked_xform (struct rte_crypto_sym_xform *xforms,
cryptodev_op_type_t op_type,
- const vnet_crypto_key_t * key)
+ const vnet_crypto_key_t *key)
{
struct rte_crypto_sym_xform *xform_cipher, *xform_auth;
vnet_crypto_key_t *key_cipher, *key_auth;
return 0;
}
-static int
-cryptodev_session_create (vnet_crypto_key_t * const key,
- struct rte_mempool *sess_priv_pool,
- cryptodev_key_t * session_pair, u32 aad_len)
-{
- struct rte_crypto_sym_xform xforms_enc[2] = { {0} };
- struct rte_crypto_sym_xform xforms_dec[2] = { {0} };
- cryptodev_main_t *cmt = &cryptodev_main;
- cryptodev_inst_t *dev_inst;
- struct rte_cryptodev *cdev;
- int ret;
- uint8_t dev_id = 0;
-
- if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
- ret = prepare_linked_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key);
- else
- ret = prepare_aead_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key,
- aad_len);
- if (ret)
- return 0;
-
- if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
- prepare_linked_xform (xforms_dec, CRYPTODEV_OP_TYPE_DECRYPT, key);
- else
- prepare_aead_xform (xforms_dec, CRYPTODEV_OP_TYPE_DECRYPT, key, aad_len);
-
- vec_foreach (dev_inst, cmt->cryptodev_inst)
- {
- dev_id = dev_inst->dev_id;
- cdev = rte_cryptodev_pmd_get_dev (dev_id);
-
- /* if the session is already configured for the driver type, avoid
- configuring it again to increase the session data's refcnt */
- if (session_pair->keys[0]->sess_data[cdev->driver_id].data &&
- session_pair->keys[1]->sess_data[cdev->driver_id].data)
- continue;
-
- ret = rte_cryptodev_sym_session_init (dev_id, session_pair->keys[0],
- xforms_enc, sess_priv_pool);
- ret = rte_cryptodev_sym_session_init (dev_id, session_pair->keys[1],
- xforms_dec, sess_priv_pool);
- if (ret < 0)
- return ret;
- }
- session_pair->keys[0]->opaque_data = aad_len;
- session_pair->keys[1]->opaque_data = aad_len;
-
- return 0;
-}
-
-static void
+static_always_inline void
cryptodev_session_del (struct rte_cryptodev_sym_session *sess)
{
u32 n_devs, i;
rte_cryptodev_sym_session_free (sess);
}
-static int
-cryptodev_check_supported_vnet_alg (vnet_crypto_key_t * key)
+static_always_inline int
+cryptodev_check_supported_vnet_alg (vnet_crypto_key_t *key)
{
vnet_crypto_alg_t alg;
if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
return -1;
}
-static_always_inline void
-cryptodev_sess_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop,
- vnet_crypto_key_index_t idx, u32 aad_len)
+static_always_inline int
+cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx,
+ u32 aad_len)
{
cryptodev_main_t *cmt = &cryptodev_main;
cryptodev_numa_data_t *numa_data;
+ cryptodev_inst_t *dev_inst;
vnet_crypto_key_t *key = vnet_crypto_get_key (idx);
struct rte_mempool *sess_pool, *sess_priv_pool;
- cryptodev_key_t *ckey = 0;
- int ret = 0;
-
- if (kop == VNET_CRYPTO_KEY_OP_DEL)
- {
- if (idx >= vec_len (cmt->keys))
- return;
-
- ckey = pool_elt_at_index (cmt->keys, idx);
- cryptodev_session_del (ckey->keys[0]);
- cryptodev_session_del (ckey->keys[1]);
- ckey->keys[0] = 0;
- ckey->keys[1] = 0;
- pool_put (cmt->keys, ckey);
- return;
- }
- else if (kop == VNET_CRYPTO_KEY_OP_MODIFY)
- {
- if (idx >= vec_len (cmt->keys))
- return;
-
- ckey = pool_elt_at_index (cmt->keys, idx);
-
- cryptodev_session_del (ckey->keys[0]);
- cryptodev_session_del (ckey->keys[1]);
- ckey->keys[0] = 0;
- ckey->keys[1] = 0;
- }
- else /* create key */
- pool_get_zero (cmt->keys, ckey);
-
- /* do not create session for unsupported alg */
- if (cryptodev_check_supported_vnet_alg (key))
- return;
+ cryptodev_key_t *ckey = vec_elt_at_index (cmt->keys, idx);
+ struct rte_crypto_sym_xform xforms_enc[2] = { { 0 } };
+ struct rte_crypto_sym_xform xforms_dec[2] = { { 0 } };
+ struct rte_cryptodev_sym_session *sessions[CRYPTODEV_N_OP_TYPES] = { 0 };
+ u32 numa_node = vm->numa_node;
+ int ret;
- numa_data = vec_elt_at_index (cmt->per_numa_data, vm->numa_node);
+ numa_data = vec_elt_at_index (cmt->per_numa_data, numa_node);
sess_pool = numa_data->sess_pool;
sess_priv_pool = numa_data->sess_priv_pool;
- ckey->keys[0] = rte_cryptodev_sym_session_create (sess_pool);
- if (!ckey->keys[0])
+ sessions[CRYPTODEV_OP_TYPE_ENCRYPT] =
+ rte_cryptodev_sym_session_create (sess_pool);
+ if (!sessions[CRYPTODEV_OP_TYPE_ENCRYPT])
{
ret = -1;
goto clear_key;
}
- ckey->keys[1] = rte_cryptodev_sym_session_create (sess_pool);
- if (!ckey->keys[1])
+ sessions[CRYPTODEV_OP_TYPE_DECRYPT] =
+ rte_cryptodev_sym_session_create (sess_pool);
+ if (!sessions[CRYPTODEV_OP_TYPE_DECRYPT])
{
ret = -1;
goto clear_key;
}
- ret = cryptodev_session_create (key, sess_priv_pool, ckey, aad_len);
+ if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
+ ret = prepare_linked_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key);
+ else
+ ret =
+ prepare_aead_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key, aad_len);
+ if (ret)
+ return 0;
+
+ if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
+ prepare_linked_xform (xforms_dec, CRYPTODEV_OP_TYPE_DECRYPT, key);
+ else
+ prepare_aead_xform (xforms_dec, CRYPTODEV_OP_TYPE_DECRYPT, key, aad_len);
+
+ vec_foreach (dev_inst, cmt->cryptodev_inst)
+ {
+ u32 dev_id = dev_inst->dev_id;
+ struct rte_cryptodev *cdev = rte_cryptodev_pmd_get_dev (dev_id);
+
+ /* if the session is already configured for the driver type, avoid
+ configuring it again to increase the session data's refcnt */
+ if (sessions[CRYPTODEV_OP_TYPE_ENCRYPT]
+ ->sess_data[cdev->driver_id]
+ .data &&
+ sessions[CRYPTODEV_OP_TYPE_DECRYPT]->sess_data[cdev->driver_id].data)
+ continue;
+
+ ret = rte_cryptodev_sym_session_init (
+ dev_id, sessions[CRYPTODEV_OP_TYPE_ENCRYPT], xforms_enc,
+ sess_priv_pool);
+ ret = rte_cryptodev_sym_session_init (
+ dev_id, sessions[CRYPTODEV_OP_TYPE_DECRYPT], xforms_dec,
+ sess_priv_pool);
+ if (ret < 0)
+ return ret;
+ }
+
+ sessions[CRYPTODEV_OP_TYPE_ENCRYPT]->opaque_data = aad_len;
+ sessions[CRYPTODEV_OP_TYPE_DECRYPT]->opaque_data = aad_len;
+
+ CLIB_MEMORY_STORE_BARRIER ();
+ ckey->keys[numa_node][CRYPTODEV_OP_TYPE_ENCRYPT] =
+ sessions[CRYPTODEV_OP_TYPE_ENCRYPT];
+ ckey->keys[numa_node][CRYPTODEV_OP_TYPE_DECRYPT] =
+ sessions[CRYPTODEV_OP_TYPE_DECRYPT];
clear_key:
if (ret != 0)
{
- cryptodev_session_del (ckey->keys[0]);
- cryptodev_session_del (ckey->keys[1]);
- memset (ckey, 0, sizeof (*ckey));
- pool_put (cmt->keys, ckey);
+ cryptodev_session_del (sessions[CRYPTODEV_OP_TYPE_ENCRYPT]);
+ cryptodev_session_del (sessions[CRYPTODEV_OP_TYPE_DECRYPT]);
+ }
+ return ret;
+}
+
+static_always_inline void
+cryptodev_sess_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop,
+ vnet_crypto_key_index_t idx, u32 aad_len)
+{
+ cryptodev_main_t *cmt = &cryptodev_main;
+ vnet_crypto_key_t *key = vnet_crypto_get_key (idx);
+ cryptodev_key_t *ckey = 0;
+ u32 i;
+
+ vec_validate (cmt->keys, idx);
+ ckey = vec_elt_at_index (cmt->keys, idx);
+
+ if (kop == VNET_CRYPTO_KEY_OP_DEL || kop == VNET_CRYPTO_KEY_OP_MODIFY)
+ {
+ if (idx >= vec_len (cmt->keys))
+ return;
+
+ vec_foreach_index (i, cmt->per_numa_data)
+ {
+ if (ckey->keys[i][CRYPTODEV_OP_TYPE_ENCRYPT])
+ {
+ cryptodev_session_del (ckey->keys[i][CRYPTODEV_OP_TYPE_ENCRYPT]);
+ cryptodev_session_del (ckey->keys[i][CRYPTODEV_OP_TYPE_DECRYPT]);
+
+ CLIB_MEMORY_STORE_BARRIER ();
+ ckey->keys[i][CRYPTODEV_OP_TYPE_ENCRYPT] = 0;
+ ckey->keys[i][CRYPTODEV_OP_TYPE_DECRYPT] = 0;
+ }
+ }
+ return;
}
+
+ /* create key */
+
+ /* do not create session for unsupported alg */
+ if (cryptodev_check_supported_vnet_alg (key))
+ return;
+
+ vec_validate (ckey->keys, vec_len (cmt->per_numa_data) - 1);
+ vec_foreach_index (i, ckey->keys)
+ vec_validate (ckey->keys[i], CRYPTODEV_N_OP_TYPES - 1);
}
/*static*/ void
cryptodev_numa_data_t *numa = cmt->per_numa_data + vm->numa_node;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
vnet_crypto_async_frame_elt_t *fe;
+ struct rte_cryptodev_sym_session *sess = 0;
cryptodev_op_t **cop;
u32 *bi;
u32 n_enqueue, n_elts;
- cryptodev_key_t *key;
- u32 last_key_index;
+ u32 last_key_index = ~0;
if (PREDICT_FALSE (frame == 0 || frame->n_elts == 0))
return -1;
cop[0]->frame = frame;
cop[0]->n_elts = n_elts;
- key = pool_elt_at_index (cmt->keys, fe->key_index);
- last_key_index = fe->key_index;
-
while (n_elts)
{
vlib_buffer_t *b = vlib_get_buffer (vm, bi[0]);
}
if (last_key_index != fe->key_index)
{
- key = pool_elt_at_index (cmt->keys, fe->key_index);
+ cryptodev_key_t *key = vec_elt_at_index (cmt->keys, fe->key_index);
last_key_index = fe->key_index;
+
+ if (key->keys[vm->numa_node][op_type] == 0)
+ {
+ if (PREDICT_FALSE (
+ cryptodev_session_create (vm, last_key_index, 0) < 0))
+ {
+ cryptodev_mark_frame_err_status (
+ frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+ return -1;
+ }
+ }
+ sess = key->keys[vm->numa_node][op_type];
}
sop->m_src = rte_mbuf_from_vlib_buffer (b);
integ_offset = 0;
crypto_offset = offset_diff;
}
- sop->session = key->keys[op_type];
+ sop->session = sess;
sop->cipher.data.offset = crypto_offset;
sop->cipher.data.length = fe->crypto_total_length;
sop->auth.data.offset = integ_offset;
cryptodev_numa_data_t *numa = cmt->per_numa_data + vm->numa_node;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
vnet_crypto_async_frame_elt_t *fe;
+ struct rte_cryptodev_sym_session *sess = 0;
cryptodev_op_t **cop;
u32 *bi;
u32 n_enqueue = 0, n_elts;
- cryptodev_key_t *key;
- u32 last_key_index;
- u8 sess_aad_len;
+ u32 last_key_index = ~0;
if (PREDICT_FALSE (frame == 0 || frame->n_elts == 0))
return -1;
cop[0]->frame = frame;
cop[0]->n_elts = n_elts;
- key = pool_elt_at_index (cmt->keys, fe->key_index);
- last_key_index = fe->key_index;
- sess_aad_len = (u8) key->keys[op_type]->opaque_data;
- if (PREDICT_FALSE (sess_aad_len != aad_len))
- cryptodev_sess_handler (vm, VNET_CRYPTO_KEY_OP_MODIFY,
- fe->key_index, aad_len);
-
while (n_elts)
{
vlib_buffer_t *b = vlib_get_buffer (vm, bi[0]);
}
if (last_key_index != fe->key_index)
{
- key = pool_elt_at_index (cmt->keys, fe->key_index);
- sess_aad_len = (u8) key->keys[op_type]->opaque_data;
- if (PREDICT_FALSE (sess_aad_len != aad_len))
+ cryptodev_key_t *key = vec_elt_at_index (cmt->keys, fe->key_index);
+
+ last_key_index = fe->key_index;
+ if (key->keys[vm->numa_node][op_type] == 0)
{
- cryptodev_sess_handler (vm, VNET_CRYPTO_KEY_OP_MODIFY,
+ if (PREDICT_FALSE (cryptodev_session_create (vm, last_key_index,
+ aad_len) < 0))
+ {
+ cryptodev_mark_frame_err_status (
+ frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+ return -1;
+ }
+ }
+ else if (PREDICT_FALSE (
+ key->keys[vm->numa_node][op_type]->opaque_data !=
+ aad_len))
+ {
+ cryptodev_sess_handler (vm, VNET_CRYPTO_KEY_OP_DEL,
fe->key_index, aad_len);
+ if (PREDICT_FALSE (cryptodev_session_create (vm, last_key_index,
+ aad_len) < 0))
+ {
+ cryptodev_mark_frame_err_status (
+ frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+ return -1;
+ }
}
- last_key_index = fe->key_index;
+
+ sess = key->keys[vm->numa_node][op_type];
}
sop->m_src = rte_mbuf_from_vlib_buffer (b);
crypto_offset = 0;
}
- sop->session = key->keys[op_type];
+ sop->session = sess;
sop->aead.aad.data = cop[0]->aad;
sop->aead.aad.phys_addr = cop[0]->op.phys_addr + CRYPTODEV_AAD_OFFSET;
sop->aead.data.length = fe->crypto_total_length;
for (i = 0; i < n_cryptodev; i++)
{
rte_cryptodev_info_get (i, &info);
- if (rte_cryptodev_socket_id (i) != numa)
- {
- clib_warning ("DPDK crypto resource %s is in different numa node "
- "as %u, ignored", info.device->name, numa);
- continue;
- }
+
/* only device support symmetric crypto is used */
if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO))
continue;
struct rte_cryptodev_info info;
struct rte_cryptodev *cdev;
cryptodev_main_t *cmt = &cryptodev_main;
- cryptodev_numa_data_t *numa_data = vec_elt_at_index (cmt->per_numa_data,
- vm->numa_node);
u32 i;
int ret;
{
struct rte_cryptodev_config cfg;
- cfg.socket_id = vm->numa_node;
+ cfg.socket_id = info.device->numa_node;
cfg.nb_queue_pairs = info.max_nb_queue_pairs;
rte_cryptodev_configure (cryptodev_id, &cfg);
int ret;
- qp_cfg.mp_session = numa_data->sess_pool;
- qp_cfg.mp_session_private = numa_data->sess_priv_pool;
+ qp_cfg.mp_session = 0;
+ qp_cfg.mp_session_private = 0;
qp_cfg.nb_descriptors = CRYPTODEV_NB_CRYPTO_OPS;
ret = rte_cryptodev_queue_pair_setup (cryptodev_id, i, &qp_cfg,
- vm->numa_node);
+ info.device->numa_node);
if (ret)
break;
}
/* A total of 4 times n_worker threads * frame size as crypto ops */
n_cop_elts = max_pow2 ((u64)n_workers * CRYPTODEV_NB_CRYPTO_OPS);
- vec_validate (cmt->per_numa_data, vm->numa_node);
- numa_data = vec_elt_at_index (cmt->per_numa_data, numa);
-
- /* create session pool for the numa node */
- name = format (0, "vcryptodev_sess_pool_%u%c", numa, 0);
- mp = rte_cryptodev_sym_session_pool_create ((char *) name,
- CRYPTODEV_NB_SESSION,
- 0, 0, 0, numa);
- if (!mp)
- {
- error = clib_error_return (0, "Not enough memory for mp %s", name);
- goto err_handling;
- }
- vec_free (name);
-
- numa_data->sess_pool = mp;
-
- /* create session private pool for the numa node */
- name = format (0, "cryptodev_sess_pool_%u%c", numa, 0);
- mp = rte_mempool_create ((char *) name, CRYPTODEV_NB_SESSION, sess_sz, 0,
- 0, NULL, NULL, NULL, NULL, numa, 0);
- if (!mp)
- {
- error = clib_error_return (0, "Not enough memory for mp %s", name);
- vec_free (name);
- goto err_handling;
- }
-
- vec_free (name);
-
- numa_data->sess_priv_pool = mp;
-
- /* create cryptodev op pool */
- name = format (0, "cryptodev_op_pool_%u%c", numa, 0);
-
- mp = rte_mempool_create ((char *) name, n_cop_elts,
- sizeof (cryptodev_op_t), VLIB_FRAME_SIZE * 2,
- sizeof (struct rte_crypto_op_pool_private), NULL,
- NULL, crypto_op_init, NULL, numa, 0);
- if (!mp)
- {
- error = clib_error_return (0, "Not enough memory for mp %s", name);
- vec_free (name);
- goto err_handling;
- }
-
- priv = rte_mempool_get_priv (mp);
- priv->priv_size = sizeof (struct rte_crypto_op_pool_private);
- priv->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
- vec_free (name);
- numa_data->cop_pool = mp;
-
/* probe all cryptodev devices and get queue info */
if (cryptodev_probe (vm, n_workers) < 0)
{
for (i = skip_master; i < tm->n_vlib_mains; i++)
{
ptd = cmt->per_thread_data + i;
+
cryptodev_assign_resource (ptd, 0, CRYPTODEV_RESOURCE_ASSIGN_AUTO);
name = format (0, "frames_ring_%u%c", i, 0);
ptd->ring = rte_ring_create((char *) name, CRYPTODEV_NB_CRYPTO_OPS,
}
vec_validate (ptd->cops, VNET_CRYPTO_FRAME_SIZE - 1);
vec_free(name);
+
+ numa = vlib_mains[i]->numa_node;
+
+ vec_validate (cmt->per_numa_data, numa);
+ numa_data = vec_elt_at_index (cmt->per_numa_data, numa);
+
+ if (numa_data->sess_pool)
+ continue;
+
+ /* create session pool for the numa node */
+ name = format (0, "vcryptodev_sess_pool_%u%c", numa, 0);
+ mp = rte_cryptodev_sym_session_pool_create (
+ (char *) name, CRYPTODEV_NB_SESSION, 0, 0, 0, numa);
+ if (!mp)
+ {
+ error = clib_error_return (0, "Not enough memory for mp %s", name);
+ goto err_handling;
+ }
+ vec_free (name);
+
+ numa_data->sess_pool = mp;
+
+ /* create session private pool for the numa node */
+ name = format (0, "cryptodev_sess_pool_%u%c", numa, 0);
+ mp = rte_mempool_create ((char *) name, CRYPTODEV_NB_SESSION, sess_sz, 0,
+ 0, NULL, NULL, NULL, NULL, numa, 0);
+ if (!mp)
+ {
+ error = clib_error_return (0, "Not enough memory for mp %s", name);
+ vec_free (name);
+ goto err_handling;
+ }
+
+ vec_free (name);
+
+ numa_data->sess_priv_pool = mp;
+
+ /* create cryptodev op pool */
+ name = format (0, "cryptodev_op_pool_%u%c", numa, 0);
+
+ mp = rte_mempool_create ((char *) name, n_cop_elts,
+ sizeof (cryptodev_op_t), VLIB_FRAME_SIZE * 2,
+ sizeof (struct rte_crypto_op_pool_private),
+ NULL, NULL, crypto_op_init, NULL, numa, 0);
+ if (!mp)
+ {
+ error = clib_error_return (0, "Not enough memory for mp %s", name);
+ vec_free (name);
+ goto err_handling;
+ }
+
+ priv = rte_mempool_get_priv (mp);
+ priv->priv_size = sizeof (struct rte_crypto_op_pool_private);
+ priv->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
+ vec_free (name);
+ numa_data->cop_pool = mp;
}
/* register handler */
- eidx = vnet_crypto_register_engine (vm, "dpdk_cryptodev", 79,
- "DPDK Cryptodev Engine");
+ eidx = vnet_crypto_register_engine (vm, "dpdk_cryptodev", 100,
+ "DPDK Cryptodev Engine");
#define _(a, b, c, d, e, f) \
vnet_crypto_register_async_handler \
vnet_crypto_register_key_handler (vm, eidx, cryptodev_key_handler);
+ /* this engine is only enabled when cryptodev device(s) are presented in
+ * startup.conf. Assume it is wanted to be used, turn on async mode here.
+ */
+ vnet_crypto_request_async_mode (1);
+ ipsec_set_async_mode (1);
+
return 0;
err_handling:
#include <vlib/vlib.h>
#include <vnet/plugin/plugin.h>
#include <vnet/crypto/crypto.h>
-#include <vnet/vnet.h>
+#include <vnet/ipsec/ipsec.h>
#include <vpp/app/version.h>
#include <dpdk/buffer.h>
typedef struct
{
- union rte_cryptodev_session_ctx keys[CRYPTODEV_N_OP_TYPES];
+ union rte_cryptodev_session_ctx **keys;
} cryptodev_key_t;
typedef struct
{
u32 dev_id;
u32 q_id;
- struct rte_crypto_raw_dp_ctx *raw_dp_ctx_buffer;
char *desc;
} cryptodev_inst_t;
u16 cryptodev_id;
u16 cryptodev_q;
u16 inflight;
+ union rte_cryptodev_session_ctx reset_sess; /* session data for reset ctx */
} cryptodev_engine_thread_t;
typedef struct
cryptodev_main_t cryptodev_main;
-static int
+static_always_inline int
prepare_aead_xform (struct rte_crypto_sym_xform *xform,
- cryptodev_op_type_t op_type,
- const vnet_crypto_key_t * key, u32 aad_len)
+ cryptodev_op_type_t op_type, const vnet_crypto_key_t *key,
+ u32 aad_len)
{
struct rte_crypto_aead_xform *aead_xform = &xform->aead;
memset (xform, 0, sizeof (*xform));
return 0;
}
-static int
+static_always_inline int
prepare_linked_xform (struct rte_crypto_sym_xform *xforms,
cryptodev_op_type_t op_type,
- const vnet_crypto_key_t * key)
+ const vnet_crypto_key_t *key)
{
struct rte_crypto_sym_xform *xform_cipher, *xform_auth;
vnet_crypto_key_t *key_cipher, *key_auth;
return 0;
}
-static int
-cryptodev_session_create (vnet_crypto_key_t * const key,
- struct rte_mempool *sess_priv_pool,
- cryptodev_key_t * session_pair, u32 aad_len)
+static_always_inline void
+cryptodev_session_del (struct rte_cryptodev_sym_session *sess)
+{
+ u32 n_devs, i;
+
+ if (sess == NULL)
+ return;
+
+ n_devs = rte_cryptodev_count ();
+
+ for (i = 0; i < n_devs; i++)
+ rte_cryptodev_sym_session_clear (i, sess);
+
+ rte_cryptodev_sym_session_free (sess);
+}
+
+static_always_inline int
+cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx,
+ u32 aad_len)
{
- struct rte_crypto_sym_xform xforms_enc[2] = { {0} };
- struct rte_crypto_sym_xform xforms_dec[2] = { {0} };
cryptodev_main_t *cmt = &cryptodev_main;
+ cryptodev_numa_data_t *numa_data;
cryptodev_inst_t *dev_inst;
- struct rte_cryptodev *cdev;
+ vnet_crypto_key_t *key = vnet_crypto_get_key (idx);
+ struct rte_mempool *sess_pool, *sess_priv_pool;
+ cryptodev_key_t *ckey = vec_elt_at_index (cmt->keys, idx);
+ struct rte_crypto_sym_xform xforms_enc[2] = { { 0 } };
+ struct rte_crypto_sym_xform xforms_dec[2] = { { 0 } };
+ struct rte_cryptodev_sym_session *sessions[CRYPTODEV_N_OP_TYPES] = { 0 };
+ u32 numa_node = vm->numa_node;
int ret;
- uint8_t dev_id = 0;
+
+ numa_data = vec_elt_at_index (cmt->per_numa_data, numa_node);
+ sess_pool = numa_data->sess_pool;
+ sess_priv_pool = numa_data->sess_priv_pool;
+
+ sessions[CRYPTODEV_OP_TYPE_ENCRYPT] =
+ rte_cryptodev_sym_session_create (sess_pool);
+ if (!sessions[CRYPTODEV_OP_TYPE_ENCRYPT])
+ {
+ ret = -1;
+ goto clear_key;
+ }
+
+ sessions[CRYPTODEV_OP_TYPE_DECRYPT] =
+ rte_cryptodev_sym_session_create (sess_pool);
+ if (!sessions[CRYPTODEV_OP_TYPE_DECRYPT])
+ {
+ ret = -1;
+ goto clear_key;
+ }
if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
ret = prepare_linked_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key);
vec_foreach (dev_inst, cmt->cryptodev_inst)
{
- dev_id = dev_inst->dev_id;
- cdev = rte_cryptodev_pmd_get_dev (dev_id);
+ u32 dev_id = dev_inst->dev_id;
+ struct rte_cryptodev *cdev = rte_cryptodev_pmd_get_dev (dev_id);
/* if the session is already configured for the driver type, avoid
configuring it again to increase the session data's refcnt */
- if (session_pair->keys[0].crypto_sess->sess_data[cdev->driver_id].data &&
- session_pair->keys[1].crypto_sess->sess_data[cdev->driver_id].data)
+ if (sessions[CRYPTODEV_OP_TYPE_ENCRYPT]->sess_data[cdev->driver_id].data &&
+ sessions[CRYPTODEV_OP_TYPE_DECRYPT]->sess_data[cdev->driver_id].data)
continue;
- ret = rte_cryptodev_sym_session_init (dev_id,
- session_pair->keys[0].crypto_sess,
- xforms_enc, sess_priv_pool);
- ret = rte_cryptodev_sym_session_init (dev_id,
- session_pair->keys[1].crypto_sess,
- xforms_dec, sess_priv_pool);
+ ret = rte_cryptodev_sym_session_init (
+ dev_id, sessions[CRYPTODEV_OP_TYPE_ENCRYPT], xforms_enc, sess_priv_pool);
+ ret = rte_cryptodev_sym_session_init (
+ dev_id, sessions[CRYPTODEV_OP_TYPE_DECRYPT], xforms_dec, sess_priv_pool);
if (ret < 0)
return ret;
}
- session_pair->keys[0].crypto_sess->opaque_data = aad_len;
- session_pair->keys[1].crypto_sess->opaque_data = aad_len;
- return 0;
-}
+ sessions[CRYPTODEV_OP_TYPE_ENCRYPT]->opaque_data = aad_len;
+ sessions[CRYPTODEV_OP_TYPE_DECRYPT]->opaque_data = aad_len;
-static void
-cryptodev_session_del (struct rte_cryptodev_sym_session *sess)
-{
- u32 n_devs, i;
-
- if (sess == NULL)
- return;
-
- n_devs = rte_cryptodev_count ();
+ CLIB_MEMORY_STORE_BARRIER ();
+ ckey->keys[numa_node][CRYPTODEV_OP_TYPE_ENCRYPT].crypto_sess =
+ sessions[CRYPTODEV_OP_TYPE_ENCRYPT];
+ ckey->keys[numa_node][CRYPTODEV_OP_TYPE_DECRYPT].crypto_sess =
+ sessions[CRYPTODEV_OP_TYPE_DECRYPT];
- for (i = 0; i < n_devs; i++)
- rte_cryptodev_sym_session_clear (i, sess);
-
- rte_cryptodev_sym_session_free (sess);
+clear_key:
+ if (ret != 0)
+ {
+ cryptodev_session_del (sessions[CRYPTODEV_OP_TYPE_ENCRYPT]);
+ cryptodev_session_del (sessions[CRYPTODEV_OP_TYPE_DECRYPT]);
+ }
+ return ret;
}
static int
vnet_crypto_key_index_t idx, u32 aad_len)
{
cryptodev_main_t *cmt = &cryptodev_main;
- cryptodev_numa_data_t *numa_data;
vnet_crypto_key_t *key = vnet_crypto_get_key (idx);
- struct rte_mempool *sess_pool, *sess_priv_pool;
cryptodev_key_t *ckey = 0;
- int ret = 0;
+ u32 i;
+
+ vec_validate (cmt->keys, idx);
+ ckey = vec_elt_at_index (cmt->keys, idx);
- if (kop == VNET_CRYPTO_KEY_OP_DEL)
+ if (kop == VNET_CRYPTO_KEY_OP_DEL || kop == VNET_CRYPTO_KEY_OP_MODIFY)
{
if (idx >= vec_len (cmt->keys))
return;
- ckey = pool_elt_at_index (cmt->keys, idx);
- cryptodev_session_del (ckey->keys[0].crypto_sess);
- cryptodev_session_del (ckey->keys[1].crypto_sess);
- ckey->keys[0].crypto_sess = 0;
- ckey->keys[1].crypto_sess = 0;
- pool_put (cmt->keys, ckey);
+ vec_foreach_index (i, cmt->per_numa_data)
+ {
+ if (ckey->keys[i][CRYPTODEV_OP_TYPE_ENCRYPT].crypto_sess)
+ {
+ cryptodev_session_del (
+ ckey->keys[i][CRYPTODEV_OP_TYPE_ENCRYPT].crypto_sess);
+ cryptodev_session_del (
+ ckey->keys[i][CRYPTODEV_OP_TYPE_DECRYPT].crypto_sess);
+
+ CLIB_MEMORY_STORE_BARRIER ();
+ ckey->keys[i][CRYPTODEV_OP_TYPE_ENCRYPT].crypto_sess = 0;
+ ckey->keys[i][CRYPTODEV_OP_TYPE_DECRYPT].crypto_sess = 0;
+ }
+ }
return;
}
- else if (kop == VNET_CRYPTO_KEY_OP_MODIFY)
- {
- if (idx >= vec_len (cmt->keys))
- return;
-
- ckey = pool_elt_at_index (cmt->keys, idx);
- cryptodev_session_del (ckey->keys[0].crypto_sess);
- cryptodev_session_del (ckey->keys[1].crypto_sess);
- ckey->keys[0].crypto_sess = 0;
- ckey->keys[1].crypto_sess = 0;
- }
- else /* create key */
- pool_get_zero (cmt->keys, ckey);
+ /* create key */
/* do not create session for unsupported alg */
if (cryptodev_check_supported_vnet_alg (key))
return;
- numa_data = vec_elt_at_index (cmt->per_numa_data, vm->numa_node);
- sess_pool = numa_data->sess_pool;
- sess_priv_pool = numa_data->sess_priv_pool;
-
- ckey->keys[0].crypto_sess = rte_cryptodev_sym_session_create (sess_pool);
- if (!ckey->keys[0].crypto_sess)
- {
- ret = -1;
- goto clear_key;
- }
-
- ckey->keys[1].crypto_sess = rte_cryptodev_sym_session_create (sess_pool);
- if (!ckey->keys[1].crypto_sess)
- {
- ret = -1;
- goto clear_key;
- }
-
- ret = cryptodev_session_create (key, sess_priv_pool, ckey, aad_len);
-
-clear_key:
- if (ret != 0)
- {
- cryptodev_session_del (ckey->keys[0].crypto_sess);
- cryptodev_session_del (ckey->keys[1].crypto_sess);
- memset (ckey, 0, sizeof (*ckey));
- pool_put (cmt->keys, ckey);
- }
+ vec_validate (ckey->keys, vec_len (cmt->per_numa_data) - 1);
+ vec_foreach_index (i, ckey->keys)
+ vec_validate (ckey->keys[i], CRYPTODEV_N_OP_TYPES - 1);
}
/*static*/ void
return ofs.raw;
}
-/* Reset cryptodev dp context to previous queue pair state */
static_always_inline void
-cryptodev_reset_ctx (u16 cdev_id, u16 qid, struct rte_crypto_raw_dp_ctx *ctx)
+cryptodev_reset_ctx (cryptodev_engine_thread_t *cet)
{
- union rte_cryptodev_session_ctx session_ctx = {.crypto_sess = NULL };
-
- rte_cryptodev_configure_raw_dp_ctx (cdev_id, qid, ctx, ~0, session_ctx, 0);
+ rte_cryptodev_configure_raw_dp_ctx (cet->cryptodev_id, cet->cryptodev_q,
+ cet->ctx, RTE_CRYPTO_OP_WITH_SESSION,
+ cet->reset_sess, 0);
}
static_always_inline int
struct rte_crypto_va_iova_ptr iv_vec, digest_vec;
vlib_buffer_t **b;
u32 n_elts;
- cryptodev_key_t *key;
u32 last_key_index = ~0;
i16 min_ofs;
u32 max_end;
if (PREDICT_FALSE (last_key_index != fe->key_index))
{
- key = pool_elt_at_index (cmt->keys, fe->key_index);
- last_key_index = fe->key_index;
+ cryptodev_key_t *key = vec_elt_at_index (cmt->keys, fe->key_index);
- if (PREDICT_FALSE
- (rte_cryptodev_configure_raw_dp_ctx
- (cet->cryptodev_id, cet->cryptodev_q, cet->ctx,
- RTE_CRYPTO_OP_WITH_SESSION, key->keys[op_type], 1) < 0))
+ if (PREDICT_FALSE (key->keys[vm->numa_node][op_type].crypto_sess ==
+ 0))
{
- cryptodev_mark_frame_err_status (frame,
- VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
- cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q,
- cet->ctx);
- return -1;
+ status = cryptodev_session_create (vm, fe->key_index, 0);
+ if (PREDICT_FALSE (status < 0))
+ goto error_exit;
}
+
+ status = rte_cryptodev_configure_raw_dp_ctx (
+ cet->cryptodev_id, cet->cryptodev_q, cet->ctx,
+ RTE_CRYPTO_OP_WITH_SESSION, key->keys[vm->numa_node][op_type],
+ /*is_update */ 1);
+ if (PREDICT_FALSE (status < 0))
+ goto error_exit;
+
+ last_key_index = fe->key_index;
}
cofs.raw = compute_ofs_linked_alg (fe, &min_ofs, &max_end);
if (PREDICT_FALSE (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS))
{
vec[0].len = b[0]->current_data + b[0]->current_length - min_ofs;
- if (cryptodev_frame_build_sgl
- (vm, cmt->iova_mode, vec, &n_seg, b[0],
- max_end - min_ofs - vec->len) < 0)
- {
- cryptodev_mark_frame_err_status (frame,
- VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
- cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q,
- cet->ctx);
- return -1;
- }
+ if (cryptodev_frame_build_sgl (vm, cmt->iova_mode, vec, &n_seg, b[0],
+ max_end - min_ofs - vec->len) < 0)
+ goto error_exit;
}
status = rte_cryptodev_raw_enqueue (cet->ctx, vec, n_seg, cofs, &iv_vec,
&digest_vec, 0, (void *) frame);
- if (status < 0)
- {
- cryptodev_mark_frame_err_status (frame,
- VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
- cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q, cet->ctx);
- return -1;
- }
+ if (PREDICT_FALSE (status < 0))
+ goto error_exit;
b++;
fe++;
status = rte_cryptodev_raw_enqueue_done (cet->ctx, frame->n_elts);
if (PREDICT_FALSE (status < 0))
{
- cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q, cet->ctx);
+ cryptodev_reset_ctx (cet);
return -1;
}
cet->inflight += frame->n_elts;
return 0;
+
+error_exit:
+ cryptodev_mark_frame_err_status (frame,
+ VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+ cryptodev_reset_ctx (cet);
+ return -1;
}
static_always_inline int
vnet_crypto_async_frame_elt_t *fe;
vlib_buffer_t **b;
u32 n_elts;
- cryptodev_key_t *key;
- u32 last_key_index = ~0;
union rte_crypto_sym_ofs cofs;
struct rte_crypto_vec *vec;
struct rte_crypto_va_iova_ptr iv_vec, digest_vec, aad_vec;
- u8 sess_aad_len = 0;
+ u32 last_key_index = ~0;
int status;
-
n_elts = frame->n_elts;
if (PREDICT_FALSE (CRYPTODEV_MAX_INFLIGHT - cet->inflight < n_elts))
vlib_prefetch_buffer_header (b[1], LOAD);
}
- if (last_key_index != fe->key_index)
+ if (PREDICT_FALSE (last_key_index != fe->key_index))
{
- key = pool_elt_at_index (cmt->keys, fe->key_index);
- sess_aad_len = (u8) key->keys[op_type].crypto_sess->opaque_data;
- if (PREDICT_FALSE (sess_aad_len != aad_len))
+ cryptodev_key_t *key = vec_elt_at_index (cmt->keys, fe->key_index);
+
+ if (PREDICT_FALSE (key->keys[vm->numa_node][op_type].crypto_sess ==
+ 0))
{
- cryptodev_sess_handler (vm, VNET_CRYPTO_KEY_OP_MODIFY,
- fe->key_index, aad_len);
+ status = cryptodev_session_create (vm, fe->key_index, aad_len);
+ if (PREDICT_FALSE (status < 0))
+ goto error_exit;
}
- last_key_index = fe->key_index;
- if (PREDICT_FALSE
- (rte_cryptodev_configure_raw_dp_ctx
- (cet->cryptodev_id, cet->cryptodev_q, cet->ctx,
- RTE_CRYPTO_OP_WITH_SESSION, key->keys[op_type], 1) < 0))
+ if (PREDICT_FALSE ((u8) key->keys[vm->numa_node][op_type]
+ .crypto_sess->opaque_data != aad_len))
{
- cryptodev_mark_frame_err_status (frame,
- VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
- cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q,
- cet->ctx);
- return -1;
+ cryptodev_sess_handler (vm, VNET_CRYPTO_KEY_OP_DEL,
+ fe->key_index, aad_len);
+ status = cryptodev_session_create (vm, fe->key_index, aad_len);
+ if (PREDICT_FALSE (status < 0))
+ goto error_exit;
}
+
+ status = rte_cryptodev_configure_raw_dp_ctx (
+ cet->cryptodev_id, cet->cryptodev_q, cet->ctx,
+ RTE_CRYPTO_OP_WITH_SESSION, key->keys[vm->numa_node][op_type],
+ /*is_update */ 1);
+ if (PREDICT_FALSE (status < 0))
+ goto error_exit;
+
+ last_key_index = fe->key_index;
}
if (cmt->iova_mode == RTE_IOVA_VA)
if (PREDICT_FALSE (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS))
{
- vec[0].len = b[0]->current_data +
- b[0]->current_length - fe->crypto_start_offset;
- if (cryptodev_frame_build_sgl
- (vm, cmt->iova_mode, vec, &n_seg, b[0],
- fe->crypto_total_length - vec[0].len) < 0)
- {
- cryptodev_mark_frame_err_status (frame,
- VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
- cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q,
- cet->ctx);
- return -1;
- }
+ vec[0].len = b[0]->current_data + b[0]->current_length -
+ fe->crypto_start_offset;
+ status =
+ cryptodev_frame_build_sgl (vm, cmt->iova_mode, vec, &n_seg, b[0],
+ fe->crypto_total_length - vec[0].len);
+ if (status < 0)
+ goto error_exit;
}
status =
- rte_cryptodev_raw_enqueue (cet->ctx, vec, n_seg, cofs,
- &iv_vec, &digest_vec, &aad_vec,
- (void *) frame);
+ rte_cryptodev_raw_enqueue (cet->ctx, vec, n_seg, cofs, &iv_vec,
+ &digest_vec, &aad_vec, (void *) frame);
if (PREDICT_FALSE (status < 0))
- {
- cryptodev_mark_frame_err_status (frame,
- VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
- cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q, cet->ctx);
- return -1;
- }
+ goto error_exit;
+
fe++;
b++;
n_elts--;
status = rte_cryptodev_raw_enqueue_done (cet->ctx, frame->n_elts);
if (PREDICT_FALSE (status < 0))
- {
- cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q, cet->ctx);
- return -1;
- }
+ goto error_exit;
cet->inflight += frame->n_elts;
return 0;
+
+error_exit:
+ cryptodev_mark_frame_err_status (frame,
+ VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+ cryptodev_reset_ctx (cet);
+ return -1;
}
static u32
cinst = vec_elt_at_index (cmt->cryptodev_inst, idx);
cet->cryptodev_id = cinst->dev_id;
cet->cryptodev_q = cinst->q_id;
- cet->ctx = cinst->raw_dp_ctx_buffer;
+ cryptodev_reset_ctx (cet);
clib_spinlock_unlock (&cmt->tlock);
break;
case CRYPTODEV_RESOURCE_ASSIGN_UPDATE:
cinst = cmt->cryptodev_inst + cryptodev_inst_index;
cet->cryptodev_id = cinst->dev_id;
cet->cryptodev_q = cinst->q_id;
- cet->ctx = cinst->raw_dp_ctx_buffer;
+ cryptodev_reset_ctx (cet);
clib_spinlock_unlock (&cmt->tlock);
break;
default:
for (i = 0; i < n_cryptodev; i++)
{
rte_cryptodev_info_get (i, &info);
- if (rte_cryptodev_socket_id (i) != numa)
- {
- clib_warning ("DPDK crypto resource %s is in different numa node "
- "as %u, ignored", info.device->name, numa);
- continue;
- }
q_count += info.max_nb_queue_pairs;
}
static int
cryptodev_configure (vlib_main_t *vm, u32 cryptodev_id)
{
+ struct rte_cryptodev_config cfg;
struct rte_cryptodev_info info;
- struct rte_cryptodev *cdev;
cryptodev_main_t *cmt = &cryptodev_main;
- cryptodev_numa_data_t *numa_data = vec_elt_at_index (cmt->per_numa_data,
- vm->numa_node);
- u32 dp_size = 0;
u32 i;
int ret;
- cdev = rte_cryptodev_pmd_get_dev (cryptodev_id);
rte_cryptodev_info_get (cryptodev_id, &info);
if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP))
ret = check_cryptodev_alg_support (cryptodev_id);
if (ret != 0)
- return ret;
+ {
+ clib_warning (
+ "Cryptodev: device %u does not support required algorithms",
+ cryptodev_id);
+ return ret;
+ }
+ cfg.socket_id = info.device->numa_node;
+ cfg.nb_queue_pairs = info.max_nb_queue_pairs;
+ rte_cryptodev_configure (cryptodev_id, &cfg);
- /** If the device is already started, we reuse it, otherwise configure
- * both the device and queue pair.
- **/
- if (!cdev->data->dev_started)
+ for (i = 0; i < info.max_nb_queue_pairs; i++)
{
- struct rte_cryptodev_config cfg;
+ struct rte_cryptodev_qp_conf qp_cfg;
- cfg.socket_id = vm->numa_node;
- cfg.nb_queue_pairs = info.max_nb_queue_pairs;
+ qp_cfg.mp_session = 0;
+ qp_cfg.mp_session_private = 0;
+ qp_cfg.nb_descriptors = CRYPTODEV_NB_CRYPTO_OPS;
- rte_cryptodev_configure (cryptodev_id, &cfg);
-
- for (i = 0; i < info.max_nb_queue_pairs; i++)
+ ret = rte_cryptodev_queue_pair_setup (cryptodev_id, i, &qp_cfg,
+ info.device->numa_node);
+ if (ret)
{
- struct rte_cryptodev_qp_conf qp_cfg;
-
- qp_cfg.mp_session = numa_data->sess_pool;
- qp_cfg.mp_session_private = numa_data->sess_priv_pool;
- qp_cfg.nb_descriptors = CRYPTODEV_NB_CRYPTO_OPS;
-
- ret = rte_cryptodev_queue_pair_setup (cryptodev_id, i, &qp_cfg,
- vm->numa_node);
- if (ret)
- break;
+ clib_warning ("Cryptodev: Configure device %u queue %u failed %d",
+ cryptodev_id, i, ret);
+ break;
}
- if (i != info.max_nb_queue_pairs)
- return -1;
-
- /* start the device */
- rte_cryptodev_start (i);
}
- ret = rte_cryptodev_get_raw_dp_ctx_size (cryptodev_id);
- if (ret < 0)
+ if (i != info.max_nb_queue_pairs)
return -1;
- dp_size = ret;
+
+ /* start the device */
+ rte_cryptodev_start (cryptodev_id);
for (i = 0; i < info.max_nb_queue_pairs; i++)
{
cdev_inst->desc = vec_new (char, strlen (info.device->name) + 10);
cdev_inst->dev_id = cryptodev_id;
cdev_inst->q_id = i;
- vec_validate_aligned (cdev_inst->raw_dp_ctx_buffer, dp_size, 8);
- cryptodev_reset_ctx (cdev_inst->dev_id, cdev_inst->q_id,
- cdev_inst->raw_dp_ctx_buffer);
snprintf (cdev_inst->desc, strlen (info.device->name) + 9,
"%s_q%u", info.device->name, i);
return 0;
}
-static int
-cryptodev_get_session_sz (vlib_main_t *vm, u32 n_workers)
+static void
+cryptodev_get_max_sz (u32 *max_sess_sz, u32 *max_dp_sz)
{
- u32 sess_data_sz = 0, i;
-
- if (rte_cryptodev_count () == 0)
- return -1;
+ cryptodev_main_t *cmt = &cryptodev_main;
+ cryptodev_inst_t *cinst;
+ u32 max_sess = 0, max_dp = 0;
- for (i = 0; i < rte_cryptodev_count (); i++)
+ vec_foreach (cinst, cmt->cryptodev_inst)
{
- u32 dev_sess_sz = rte_cryptodev_sym_get_private_session_size (i);
+ u32 sess_sz = rte_cryptodev_sym_get_private_session_size (cinst->dev_id);
+ u32 dp_sz = rte_cryptodev_get_raw_dp_ctx_size (cinst->dev_id);
- sess_data_sz = dev_sess_sz > sess_data_sz ? dev_sess_sz : sess_data_sz;
+ max_sess = clib_max (sess_sz, max_sess);
+ max_dp = clib_max (dp_sz, max_dp);
}
- return sess_data_sz;
+ *max_sess_sz = max_sess;
+ *max_dp_sz = max_dp;
}
static void
rte_free (ptd->aad_buf);
if (ptd->cached_frame)
rte_ring_free (ptd->cached_frame);
+ if (ptd->reset_sess.crypto_sess)
+ {
+ struct rte_mempool *mp =
+ rte_mempool_from_obj ((void *) ptd->reset_sess.crypto_sess);
+
+ rte_mempool_free (mp);
+ ptd->reset_sess.crypto_sess = 0;
+ }
+ }
+}
+
+static clib_error_t *
+create_reset_sess (cryptodev_engine_thread_t *ptd, u32 lcore, u32 numa,
+ u32 sess_sz)
+{
+ struct rte_crypto_sym_xform xform = { 0 };
+ struct rte_crypto_aead_xform *aead_xform = &xform.aead;
+ struct rte_cryptodev_sym_session *sess;
+ struct rte_mempool *mp = 0;
+ u8 key[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+ u8 *name = 0;
+ clib_error_t *error = 0;
+
+ /* create session pool for the numa node */
+ name = format (0, "vcryptodev_s_reset_%u_%u", numa, lcore);
+ mp = rte_cryptodev_sym_session_pool_create ((char *) name, 2, sess_sz, 0, 0,
+ numa);
+ if (!mp)
+ {
+ error = clib_error_return (0, "Not enough memory for mp %s", name);
+ goto error_exit;
+ }
+ vec_free (name);
+
+ xform.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+ aead_xform->algo = RTE_CRYPTO_AEAD_AES_GCM;
+ aead_xform->op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
+ aead_xform->aad_length = 8;
+ aead_xform->digest_length = 16;
+ aead_xform->iv.offset = 0;
+ aead_xform->iv.length = 12;
+ aead_xform->key.data = key;
+ aead_xform->key.length = 16;
+
+ sess = rte_cryptodev_sym_session_create (mp);
+ if (!sess)
+ {
+ error = clib_error_return (0, "failed to create session");
+ goto error_exit;
+ }
+
+ if (rte_cryptodev_sym_session_init (ptd->cryptodev_id, sess, &xform, mp) < 0)
+ {
+ error = clib_error_return (0, "failed to create session private");
+ goto error_exit;
}
+
+ ptd->reset_sess.crypto_sess = sess;
+
+ return 0;
+
+error_exit:
+ if (mp)
+ rte_mempool_free (mp);
+ if (name)
+ vec_free (name);
+
+ return error;
}
clib_error_t *
u32 skip_master = vlib_num_workers () > 0;
u32 n_workers = tm->n_vlib_mains - skip_master;
u32 numa = vm->numa_node;
- i32 sess_sz;
+ u32 sess_sz, dp_sz;
u32 eidx;
u32 i;
u8 *name = 0;
cmt->iova_mode = rte_eal_iova_mode ();
- sess_sz = cryptodev_get_session_sz(vm, n_workers);
- if (sess_sz < 0)
- {
- error = clib_error_return (0, "Not enough cryptodevs");
- return error;
- }
-
vec_validate (cmt->per_numa_data, vm->numa_node);
- numa_data = vec_elt_at_index (cmt->per_numa_data, numa);
-
- /* create session pool for the numa node */
- name = format (0, "vcryptodev_sess_pool_%u%c", numa, 0);
- mp = rte_cryptodev_sym_session_pool_create ((char *) name,
- CRYPTODEV_NB_SESSION,
- 0, 0, 0, numa);
- if (!mp)
- {
- error = clib_error_return (0, "Not enough memory for mp %s", name);
- goto err_handling;
- }
- vec_free (name);
-
- numa_data->sess_pool = mp;
-
- /* create session private pool for the numa node */
- name = format (0, "cryptodev_sess_pool_%u%c", numa, 0);
- mp = rte_mempool_create ((char *) name, CRYPTODEV_NB_SESSION, sess_sz, 0,
- 0, NULL, NULL, NULL, NULL, numa, 0);
- if (!mp)
- {
- error = clib_error_return (0, "Not enough memory for mp %s", name);
- vec_free (name);
- goto err_handling;
- }
-
- vec_free (name);
-
- numa_data->sess_priv_pool = mp;
/* probe all cryptodev devices and get queue info */
if (cryptodev_probe (vm, n_workers) < 0)
goto err_handling;
}
+ cryptodev_get_max_sz (&sess_sz, &dp_sz);
+
clib_bitmap_vec_validate (cmt->active_cdev_inst_mask, tm->n_vlib_mains);
clib_spinlock_init (&cmt->tlock);
for (i = skip_master; i < tm->n_vlib_mains; i++)
{
ptd = cmt->per_thread_data + i;
- cryptodev_assign_resource (ptd, 0, CRYPTODEV_RESOURCE_ASSIGN_AUTO);
+ numa = vlib_mains[i]->numa_node;
+
ptd->aad_buf = rte_zmalloc_socket (0, CRYPTODEV_NB_CRYPTO_OPS *
CRYPTODEV_MAX_AAD_SIZE,
CLIB_CACHE_LINE_BYTES,
numa);
+
if (ptd->aad_buf == 0)
{
error = clib_error_return (0, "Failed to alloc aad buf");
ptd->aad_phy_addr = rte_malloc_virt2iova (ptd->aad_buf);
+ ptd->ctx = rte_zmalloc_socket (0, dp_sz, CLIB_CACHE_LINE_BYTES, numa);
+ if (!ptd->ctx)
+ {
+ error = clib_error_return (0, "Failed to alloc raw dp ctx");
+ goto err_handling;
+ }
+
name = format (0, "cache_frame_ring_%u%u", numa, i);
ptd->cached_frame = rte_ring_create ((char *)name,
CRYPTODEV_DEQ_CACHE_SZ, numa,
if (ptd->cached_frame == 0)
{
- error = clib_error_return (0, "Failed to frame ring");
+ error = clib_error_return (0, "Failed to alloc frame ring");
goto err_handling;
}
vec_free (name);
+
+ vec_validate (cmt->per_numa_data, numa);
+ numa_data = vec_elt_at_index (cmt->per_numa_data, numa);
+
+ if (!numa_data->sess_pool)
+ {
+ /* create session pool for the numa node */
+ name = format (0, "vcryptodev_sess_pool_%u%c", numa, 0);
+ mp = rte_cryptodev_sym_session_pool_create (
+ (char *) name, CRYPTODEV_NB_SESSION, 0, 0, 0, numa);
+ if (!mp)
+ {
+ error =
+ clib_error_return (0, "Not enough memory for mp %s", name);
+ goto err_handling;
+ }
+ vec_free (name);
+
+ numa_data->sess_pool = mp;
+
+ /* create session private pool for the numa node */
+ name = format (0, "cryptodev_sess_pool_%u%c", numa, 0);
+ mp =
+ rte_mempool_create ((char *) name, CRYPTODEV_NB_SESSION, sess_sz,
+ 0, 0, NULL, NULL, NULL, NULL, numa, 0);
+ if (!mp)
+ {
+ error =
+ clib_error_return (0, "Not enough memory for mp %s", name);
+ vec_free (name);
+ goto err_handling;
+ }
+
+ vec_free (name);
+
+ numa_data->sess_priv_pool = mp;
+ }
+
+ error = create_reset_sess (ptd, i, numa, sess_sz);
+ if (error)
+ goto err_handling;
+
+ cryptodev_assign_resource (ptd, 0, CRYPTODEV_RESOURCE_ASSIGN_AUTO);
}
/* register handler */
- eidx = vnet_crypto_register_engine (vm, "dpdk_cryptodev", 79,
- "DPDK Cryptodev Engine");
+ eidx = vnet_crypto_register_engine (vm, "dpdk_cryptodev", 100,
+ "DPDK Cryptodev Engine");
#define _(a, b, c, d, e, f) \
vnet_crypto_register_async_handler \
vnet_crypto_register_key_handler (vm, eidx, cryptodev_key_handler);
+ /* this engine is only enabled when cryptodev device(s) are presented in
+ * startup.conf. Assume it is wanted to be used, turn on async mode here.
+ */
+ vnet_crypto_request_async_mode (1);
+ ipsec_set_async_mode (1);
+
return 0;
err_handling:
Code Review / vpp.git / commitdiff