typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- MB_MGR *mgr;
__m128i cbc_iv;
+ MB_MGR *mgr;
+ JOB_AES_HMAC burst_jobs[IMB_MAX_BURST_SIZE];
} ipsecmb_per_thread_data_t;
typedef struct
ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data,
vm->thread_index);
JOB_AES_HMAC *job;
- u32 i, n_fail = 0;
+ u32 i, n_fail = 0, ops_index = 0;
u8 scratch[n_ops][digest_size];
+ const u32 burst_sz =
+ (n_ops > IMB_MAX_BURST_SIZE) ? IMB_MAX_BURST_SIZE : n_ops;
- /*
- * queue all the jobs first ...
- */
- for (i = 0; i < n_ops; i++)
+ while (n_ops)
{
- vnet_crypto_op_t *op = ops[i];
- u8 *kd = (u8 *) imbm->key_data[op->key_index];
+ const u32 n = (n_ops > burst_sz) ? burst_sz : n_ops;
+ /*
+ * configure all the jobs first ...
+ */
+ for (i = 0; i < n; i++, ops_index++)
+ {
+ vnet_crypto_op_t *op = ops[ops_index];
+ const u8 *kd = (u8 *) imbm->key_data[op->key_index];
- job = IMB_GET_NEXT_JOB (ptd->mgr);
+ job = &ptd->burst_jobs[i];
- job->src = op->src;
- job->hash_start_src_offset_in_bytes = 0;
- job->msg_len_to_hash_in_bytes = op->len;
- job->hash_alg = alg;
- job->auth_tag_output_len_in_bytes = digest_size;
- job->auth_tag_output = scratch[i];
+ job->src = op->src;
+ job->hash_start_src_offset_in_bytes = 0;
+ job->msg_len_to_hash_in_bytes = op->len;
+ job->auth_tag_output_len_in_bytes = digest_size;
+ job->auth_tag_output = scratch[ops_index];
- job->cipher_mode = NULL_CIPHER;
- job->cipher_direction = DECRYPT;
- job->chain_order = HASH_CIPHER;
+ job->u.HMAC._hashed_auth_key_xor_ipad = kd;
+ job->u.HMAC._hashed_auth_key_xor_opad = kd + hash_size;
+ job->user_data = op;
+ }
- job->u.HMAC._hashed_auth_key_xor_ipad = kd;
- job->u.HMAC._hashed_auth_key_xor_opad = kd + hash_size;
- job->user_data = op;
+ /*
+ * submit all jobs to be processed and retire completed jobs
+ */
+ IMB_SUBMIT_HASH_BURST_NOCHECK (ptd->mgr, ptd->burst_jobs, n, alg);
- job = IMB_SUBMIT_JOB (ptd->mgr);
+ for (i = 0; i < n; i++)
+ {
+ job = &ptd->burst_jobs[i];
+ ipsecmb_retire_hmac_job (job, &n_fail, digest_size);
+ }
- if (job)
- ipsecmb_retire_hmac_job (job, &n_fail, digest_size);
+ n_ops -= n;
}
- while ((job = IMB_FLUSH_JOB (ptd->mgr)))
- ipsecmb_retire_hmac_job (job, &n_fail, digest_size);
-
- return n_ops - n_fail;
+ return ops_index - n_fail;
}
#define _(a, b, c, d, e, f) \
ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data,
vm->thread_index);
JOB_AES_HMAC *job;
- u32 i, n_fail = 0;
+ u32 i, n_fail = 0, ops_index = 0;
+ const u32 burst_sz =
+ (n_ops > IMB_MAX_BURST_SIZE) ? IMB_MAX_BURST_SIZE : n_ops;
- for (i = 0; i < n_ops; i++)
+ while (n_ops)
{
- ipsecmb_aes_key_data_t *kd;
- vnet_crypto_op_t *op = ops[i];
- kd = (ipsecmb_aes_key_data_t *) imbm->key_data[op->key_index];
- __m128i iv;
+ const u32 n = (n_ops > burst_sz) ? burst_sz : n_ops;
- job = IMB_GET_NEXT_JOB (ptd->mgr);
+ for (i = 0; i < n; i++)
+ {
+ ipsecmb_aes_key_data_t *kd;
+ vnet_crypto_op_t *op = ops[ops_index++];
+ kd = (ipsecmb_aes_key_data_t *) imbm->key_data[op->key_index];
- job->src = op->src;
- job->dst = op->dst;
- job->msg_len_to_cipher_in_bytes = op->len;
- job->cipher_start_src_offset_in_bytes = 0;
+ job = &ptd->burst_jobs[i];
- job->hash_alg = NULL_HASH;
- job->cipher_mode = cipher_mode;
- job->cipher_direction = direction;
- job->chain_order = (direction == ENCRYPT ? CIPHER_HASH : HASH_CIPHER);
+ job->src = op->src;
+ job->dst = op->dst;
+ job->msg_len_to_cipher_in_bytes = op->len;
+ job->cipher_start_src_offset_in_bytes = 0;
- if ((direction == ENCRYPT) && (op->flags & VNET_CRYPTO_OP_FLAG_INIT_IV))
- {
- iv = ptd->cbc_iv;
- _mm_storeu_si128 ((__m128i *) op->iv, iv);
- ptd->cbc_iv = _mm_aesenc_si128 (iv, iv);
- }
+ job->hash_alg = NULL_HASH;
- job->aes_key_len_in_bytes = key_len / 8;
- job->aes_enc_key_expanded = kd->enc_key_exp;
- job->aes_dec_key_expanded = kd->dec_key_exp;
- job->iv = op->iv;
- job->iv_len_in_bytes = AES_BLOCK_SIZE;
+ if ((direction == ENCRYPT) &&
+ (op->flags & VNET_CRYPTO_OP_FLAG_INIT_IV))
+ {
+ const __m128i iv = ptd->cbc_iv;
+ _mm_storeu_si128 ((__m128i *) op->iv, iv);
+ ptd->cbc_iv = _mm_aesenc_si128 (iv, iv);
+ }
- job->user_data = op;
+ job->aes_enc_key_expanded = kd->enc_key_exp;
+ job->aes_dec_key_expanded = kd->dec_key_exp;
+ job->iv = op->iv;
+ job->iv_len_in_bytes = AES_BLOCK_SIZE;
- job = IMB_SUBMIT_JOB (ptd->mgr);
+ job->user_data = op;
+ }
- if (job)
- ipsecmb_retire_cipher_job (job, &n_fail);
- }
+ IMB_SUBMIT_CIPHER_BURST_NOCHECK (ptd->mgr, ptd->burst_jobs, n,
+ cipher_mode, direction, key_len / 8);
+ for (i = 0; i < n; i++)
+ {
+ job = &ptd->burst_jobs[i];
+ ipsecmb_retire_cipher_job (job, &n_fail);
+ }
- while ((job = IMB_FLUSH_JOB (ptd->mgr)))
- ipsecmb_retire_cipher_job (job, &n_fail);
+ n_ops -= n;
+ }
- return n_ops - n_fail;
+ return ops_index - n_fail;
}
#define _(a, b, c) \
foreach_ipsecmb_gcm_cipher_op;
#undef _
+#ifdef HAVE_IPSECMB_CHACHA_POLY
always_inline void
ipsecmb_retire_aead_job (JOB_AES_HMAC *job, u32 *n_fail)
{
ipsecmb_main_t *imbm = &ipsecmb_main;
ipsecmb_per_thread_data_t *ptd =
vec_elt_at_index (imbm->per_thread_data, vm->thread_index);
- struct IMB_JOB *job;
+ struct JOB_AES_HMAC *job;
MB_MGR *m = ptd->mgr;
u32 i, n_fail = 0, last_key_index = ~0;
u8 scratch[VLIB_FRAME_SIZE][16];
return ipsecmb_ops_chacha_poly_chained (vm, ops, chunks, n_ops,
IMB_DIR_DECRYPT);
}
+#endif
clib_error_t *
crypto_ipsecmb_iv_init (ipsecmb_main_t * imbm)
MB_MGR *m = 0;
u32 eidx;
u8 *name;
+ const u32 burst_jobs_sz = sizeof (JOB_AES_HMAC) * IMB_MAX_BURST_SIZE;
if (!clib_cpu_supports_aes ())
return 0;
vec_foreach (ptd, imbm->per_thread_data)
{
ptd->mgr = alloc_mb_mgr (0);
- if (clib_cpu_supports_avx512f ())
+ memset (ptd->burst_jobs, 0, burst_jobs_sz);
+
+ if (clib_cpu_supports_avx512f ())
init_mb_mgr_avx512 (ptd->mgr);
else if (clib_cpu_supports_avx2 ())
init_mb_mgr_avx2 (ptd->mgr);
foreach_ipsecmb_gcm_cipher_op;
#undef _
+#ifdef HAVE_IPSECMB_CHACHA_POLY
vnet_crypto_register_ops_handler (vm, eidx,
VNET_CRYPTO_OP_CHACHA20_POLY1305_ENC,
ipsecmb_ops_chacha_poly_enc);
ipsec_mb_ops_chacha_poly_dec_chained);
ad = imbm->alg_data + VNET_CRYPTO_ALG_CHACHA20_POLY1305;
ad->data_size = 0;
+#endif
vnet_crypto_register_key_handler (vm, eidx, crypto_ipsecmb_key_handler);
return (NULL);
Code Review / vpp.git / blobdiff