From 2a6f35f24c6ab5aba63c0dfa21e865a8cd89f7cf Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 5 Oct 2022 12:08:05 +0100 Subject: [PATCH] crypto-ipsecmb: bump ipsecmb library to v1.3 - Use the latest IPsec Multi-Buffer library release v1.3 - Use ipsec-mb burst API for HMAC-SHAx algorithms - Use ipsec-mb burst API for AES-CBC and AES-CTR algorithms The new burst API available in ipsecmb v1.3 brings significant performance improvements for certain algorithms compared to the job API. Type: feature Signed-off-by: marcel.d.cornu@intel.com Change-Id: I3490b35a616a2ea77607f103426df62438c22b2b --- build/external/packages/ipsec-mb.mk | 5 +- src/plugins/crypto_ipsecmb/ipsecmb.c | 143 +++++++++++++++++++---------------- 2 files changed, 82 insertions(+), 66 deletions(-) diff --git a/build/external/packages/ipsec-mb.mk b/build/external/packages/ipsec-mb.mk index 8750dc2d493..02c2e194456 100644 --- a/build/external/packages/ipsec-mb.mk +++ b/build/external/packages/ipsec-mb.mk @@ -11,17 +11,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -ipsec-mb_version := 1.2 +ipsec-mb_version := 1.3 ipsec-mb_tarball := v$(ipsec-mb_version).tar.gz ipsec-mb_tarball_md5sum_0.54 := 258941f7ba90c275fcf9d19c622d2d21 ipsec-mb_tarball_md5sum_0.55 := deca674bca7ae2282890e1fa7f953609 ipsec-mb_tarball_md5sum_1.0 := 906e701937751e761671dc83a41cff65 ipsec-mb_tarball_md5sum_1.1 := 3916471d3713d27e42473cb6af9c65e5 ipsec-mb_tarball_md5sum_1.2 := f551d9c208893a436c1f5c146a615bd6 +ipsec-mb_tarball_md5sum_1.3 := d8692db9efe32a263b61f12ac0dca950 ipsec-mb_tarball_md5sum := $(ipsec-mb_tarball_md5sum_$(ipsec-mb_version)) ipsec-mb_tarball_strip_dirs := 1 -ipsec-mb_url := http://github.com/01org/intel-ipsec-mb/archive/$(ipsec-mb_tarball) +ipsec-mb_url := http://github.com/intel/intel-ipsec-mb/archive/$(ipsec-mb_tarball) define ipsec-mb_config_cmds @true diff --git a/src/plugins/crypto_ipsecmb/ipsecmb.c b/src/plugins/crypto_ipsecmb/ipsecmb.c index 93654daee51..c75a2b8b6aa 100644 --- a/src/plugins/crypto_ipsecmb/ipsecmb.c +++ b/src/plugins/crypto_ipsecmb/ipsecmb.c @@ -31,8 +31,9 @@ typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - MB_MGR *mgr; __m128i cbc_iv; + MB_MGR *mgr; + JOB_AES_HMAC burst_jobs[IMB_MAX_BURST_SIZE]; } ipsecmb_per_thread_data_t; typedef struct @@ -148,44 +149,50 @@ ipsecmb_ops_hmac_inline (vlib_main_t * vm, vnet_crypto_op_t * ops[], ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, vm->thread_index); JOB_AES_HMAC *job; - u32 i, n_fail = 0; + u32 i, n_fail = 0, ops_index = 0; u8 scratch[n_ops][digest_size]; + const u32 burst_sz = + (n_ops > IMB_MAX_BURST_SIZE) ? IMB_MAX_BURST_SIZE : n_ops; - /* - * queue all the jobs first ... - */ - for (i = 0; i < n_ops; i++) + while (n_ops) { - vnet_crypto_op_t *op = ops[i]; - u8 *kd = (u8 *) imbm->key_data[op->key_index]; + const u32 n = (n_ops > burst_sz) ? burst_sz : n_ops; + /* + * configure all the jobs first ... + */ + for (i = 0; i < n; i++, ops_index++) + { + vnet_crypto_op_t *op = ops[ops_index]; + const u8 *kd = (u8 *) imbm->key_data[op->key_index]; - job = IMB_GET_NEXT_JOB (ptd->mgr); + job = &ptd->burst_jobs[i]; - job->src = op->src; - job->hash_start_src_offset_in_bytes = 0; - job->msg_len_to_hash_in_bytes = op->len; - job->hash_alg = alg; - job->auth_tag_output_len_in_bytes = digest_size; - job->auth_tag_output = scratch[i]; + job->src = op->src; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = op->len; + job->auth_tag_output_len_in_bytes = digest_size; + job->auth_tag_output = scratch[ops_index]; - job->cipher_mode = NULL_CIPHER; - job->cipher_direction = DECRYPT; - job->chain_order = HASH_CIPHER; + job->u.HMAC._hashed_auth_key_xor_ipad = kd; + job->u.HMAC._hashed_auth_key_xor_opad = kd + hash_size; + job->user_data = op; + } - job->u.HMAC._hashed_auth_key_xor_ipad = kd; - job->u.HMAC._hashed_auth_key_xor_opad = kd + hash_size; - job->user_data = op; + /* + * submit all jobs to be processed and retire completed jobs + */ + IMB_SUBMIT_HASH_BURST_NOCHECK (ptd->mgr, ptd->burst_jobs, n, alg); - job = IMB_SUBMIT_JOB (ptd->mgr); + for (i = 0; i < n; i++) + { + job = &ptd->burst_jobs[i]; + ipsecmb_retire_hmac_job (job, &n_fail, digest_size); + } - if (job) - ipsecmb_retire_hmac_job (job, &n_fail, digest_size); + n_ops -= n; } - while ((job = IMB_FLUSH_JOB (ptd->mgr))) - ipsecmb_retire_hmac_job (job, &n_fail, digest_size); - - return n_ops - n_fail; + return ops_index - n_fail; } #define _(a, b, c, d, e, f) \ @@ -222,52 +229,57 @@ ipsecmb_ops_aes_cipher_inline (vlib_main_t *vm, vnet_crypto_op_t *ops[], ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, vm->thread_index); JOB_AES_HMAC *job; - u32 i, n_fail = 0; + u32 i, n_fail = 0, ops_index = 0; + const u32 burst_sz = + (n_ops > IMB_MAX_BURST_SIZE) ? IMB_MAX_BURST_SIZE : n_ops; - for (i = 0; i < n_ops; i++) + while (n_ops) { - ipsecmb_aes_key_data_t *kd; - vnet_crypto_op_t *op = ops[i]; - kd = (ipsecmb_aes_key_data_t *) imbm->key_data[op->key_index]; - __m128i iv; + const u32 n = (n_ops > burst_sz) ? burst_sz : n_ops; - job = IMB_GET_NEXT_JOB (ptd->mgr); + for (i = 0; i < n; i++) + { + ipsecmb_aes_key_data_t *kd; + vnet_crypto_op_t *op = ops[ops_index++]; + kd = (ipsecmb_aes_key_data_t *) imbm->key_data[op->key_index]; - job->src = op->src; - job->dst = op->dst; - job->msg_len_to_cipher_in_bytes = op->len; - job->cipher_start_src_offset_in_bytes = 0; + job = &ptd->burst_jobs[i]; - job->hash_alg = NULL_HASH; - job->cipher_mode = cipher_mode; - job->cipher_direction = direction; - job->chain_order = (direction == ENCRYPT ? CIPHER_HASH : HASH_CIPHER); + job->src = op->src; + job->dst = op->dst; + job->msg_len_to_cipher_in_bytes = op->len; + job->cipher_start_src_offset_in_bytes = 0; - if ((direction == ENCRYPT) && (op->flags & VNET_CRYPTO_OP_FLAG_INIT_IV)) - { - iv = ptd->cbc_iv; - _mm_storeu_si128 ((__m128i *) op->iv, iv); - ptd->cbc_iv = _mm_aesenc_si128 (iv, iv); - } + job->hash_alg = NULL_HASH; - job->aes_key_len_in_bytes = key_len / 8; - job->aes_enc_key_expanded = kd->enc_key_exp; - job->aes_dec_key_expanded = kd->dec_key_exp; - job->iv = op->iv; - job->iv_len_in_bytes = AES_BLOCK_SIZE; + if ((direction == ENCRYPT) && + (op->flags & VNET_CRYPTO_OP_FLAG_INIT_IV)) + { + const __m128i iv = ptd->cbc_iv; + _mm_storeu_si128 ((__m128i *) op->iv, iv); + ptd->cbc_iv = _mm_aesenc_si128 (iv, iv); + } - job->user_data = op; + job->aes_enc_key_expanded = kd->enc_key_exp; + job->aes_dec_key_expanded = kd->dec_key_exp; + job->iv = op->iv; + job->iv_len_in_bytes = AES_BLOCK_SIZE; - job = IMB_SUBMIT_JOB (ptd->mgr); + job->user_data = op; + } - if (job) - ipsecmb_retire_cipher_job (job, &n_fail); - } + IMB_SUBMIT_CIPHER_BURST_NOCHECK (ptd->mgr, ptd->burst_jobs, n, + cipher_mode, direction, key_len / 8); + for (i = 0; i < n; i++) + { + job = &ptd->burst_jobs[i]; + ipsecmb_retire_cipher_job (job, &n_fail); + } - while ((job = IMB_FLUSH_JOB (ptd->mgr))) - ipsecmb_retire_cipher_job (job, &n_fail); + n_ops -= n; + } - return n_ops - n_fail; + return ops_index - n_fail; } #define _(a, b, c) \ @@ -462,7 +474,7 @@ ipsecmb_ops_chacha_poly (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, ipsecmb_main_t *imbm = &ipsecmb_main; ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, vm->thread_index); - struct IMB_JOB *job; + struct JOB_AES_HMAC *job; MB_MGR *m = ptd->mgr; u32 i, n_fail = 0, last_key_index = ~0; u8 scratch[VLIB_FRAME_SIZE][16]; @@ -779,6 +791,7 @@ crypto_ipsecmb_init (vlib_main_t * vm) MB_MGR *m = 0; u32 eidx; u8 *name; + const u32 burst_jobs_sz = sizeof (JOB_AES_HMAC) * IMB_MAX_BURST_SIZE; if (!clib_cpu_supports_aes ()) return 0; @@ -797,7 +810,9 @@ crypto_ipsecmb_init (vlib_main_t * vm) vec_foreach (ptd, imbm->per_thread_data) { ptd->mgr = alloc_mb_mgr (0); - if (clib_cpu_supports_avx512f ()) + memset (ptd->burst_jobs, 0, burst_jobs_sz); + + if (clib_cpu_supports_avx512f ()) init_mb_mgr_avx512 (ptd->mgr); else if (clib_cpu_supports_avx2 ()) init_mb_mgr_avx2 (ptd->mgr); -- 2.16.6