X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fcrypto_ipsecmb%2Fipsecmb.c;h=11e52322775556d23999d275deea3e1e9abbd164;hb=2fc4091319bdbbac25ce1132cfb73b5077426b75;hp=7495452eb29300ef256129e5af478bd35a95713c;hpb=c74009dce1b2f1466112775a68a5608d754c7c76;p=vpp.git diff --git a/src/plugins/crypto_ipsecmb/ipsecmb.c b/src/plugins/crypto_ipsecmb/ipsecmb.c index 7495452eb29..11e52322775 100644 --- a/src/plugins/crypto_ipsecmb/ipsecmb.c +++ b/src/plugins/crypto_ipsecmb/ipsecmb.c @@ -261,101 +261,142 @@ ipsecmb_ops_cbc_cipher_dec_##a (vlib_main_t * vm, \ foreach_ipsecmb_cbc_cipher_op; #undef _ -always_inline void -ipsecmb_retire_gcm_cipher_job (JOB_AES_HMAC * job, - u32 * n_fail, JOB_CIPHER_DIRECTION direction) -{ - vnet_crypto_op_t *op = job->user_data; - - if (STS_COMPLETED != job->status) - { - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; - *n_fail = *n_fail + 1; - return; - } - else - op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; - - if (DECRYPT == direction) - { - if ((memcmp (op->tag, job->auth_tag_output, op->tag_len))) - { - *n_fail = *n_fail + 1; - op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; - } - } -} - -static_always_inline u32 -ipsecmb_ops_gcm_cipher_inline (vlib_main_t * vm, vnet_crypto_op_t * ops[], - u32 n_ops, u32 key_len, - JOB_CIPHER_DIRECTION direction) -{ - ipsecmb_main_t *imbm = &ipsecmb_main; - ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, - vm->thread_index); - JOB_AES_HMAC *job; - u32 i, n_fail = 0; - u8 scratch[n_ops][64]; - - /* - * queue all the jobs first ... - */ - for (i = 0; i < n_ops; i++) - { - struct gcm_key_data *kd; - vnet_crypto_op_t *op = ops[i]; - kd = (struct gcm_key_data *) imbm->key_data[op->key_index]; - - job = IMB_GET_NEXT_JOB (ptd->mgr); - - job->src = op->src; - job->dst = op->dst; - job->msg_len_to_cipher_in_bytes = op->len; - job->cipher_start_src_offset_in_bytes = 0; - - job->hash_alg = AES_GMAC; - job->cipher_mode = GCM; - job->cipher_direction = direction; - job->chain_order = (direction == ENCRYPT ? CIPHER_HASH : HASH_CIPHER); - - job->iv = op->iv; - job->aes_key_len_in_bytes = key_len / 8; - job->aes_enc_key_expanded = kd; - job->aes_dec_key_expanded = kd; - job->iv_len_in_bytes = 12; - - job->u.GCM.aad = op->aad; - job->u.GCM.aad_len_in_bytes = op->aad_len; - job->auth_tag_output_len_in_bytes = op->tag_len; - if (DECRYPT == direction) - job->auth_tag_output = scratch[i]; - else - job->auth_tag_output = op->tag; - job->user_data = op; - - job = IMB_SUBMIT_JOB (ptd->mgr); - - if (job) - ipsecmb_retire_gcm_cipher_job (job, &n_fail, direction); - } - - while ((job = IMB_FLUSH_JOB (ptd->mgr))) - ipsecmb_retire_gcm_cipher_job (job, &n_fail, direction); - - return n_ops - n_fail; -} - #define _(a, b) \ static_always_inline u32 \ +ipsecmb_ops_gcm_cipher_enc_##a##_chained (vlib_main_t * vm, \ + vnet_crypto_op_t * ops[], vnet_crypto_op_chunk_t *chunks, u32 n_ops) \ +{ \ + ipsecmb_main_t *imbm = &ipsecmb_main; \ + ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, \ + vm->thread_index); \ + MB_MGR *m = ptd->mgr; \ + vnet_crypto_op_chunk_t *chp; \ + u32 i, j; \ + \ + for (i = 0; i < n_ops; i++) \ + { \ + struct gcm_key_data *kd; \ + struct gcm_context_data ctx; \ + vnet_crypto_op_t *op = ops[i]; \ + \ + kd = (struct gcm_key_data *) imbm->key_data[op->key_index]; \ + ASSERT (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS); \ + IMB_AES##b##_GCM_INIT(m, kd, &ctx, op->iv, op->aad, op->aad_len); \ + chp = chunks + op->chunk_index; \ + for (j = 0; j < op->n_chunks; j++) \ + { \ + IMB_AES##b##_GCM_ENC_UPDATE (m, kd, &ctx, chp->dst, chp->src, \ + chp->len); \ + chp += 1; \ + } \ + IMB_AES##b##_GCM_ENC_FINALIZE(m, kd, &ctx, op->tag, op->tag_len); \ + \ + op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; \ + } \ + \ + return n_ops; \ +} \ + \ +static_always_inline u32 \ ipsecmb_ops_gcm_cipher_enc_##a (vlib_main_t * vm, vnet_crypto_op_t * ops[], \ u32 n_ops) \ -{ return ipsecmb_ops_gcm_cipher_inline (vm, ops, n_ops, b, ENCRYPT); } \ +{ \ + ipsecmb_main_t *imbm = &ipsecmb_main; \ + ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, \ + vm->thread_index); \ + MB_MGR *m = ptd->mgr; \ + u32 i; \ + \ + for (i = 0; i < n_ops; i++) \ + { \ + struct gcm_key_data *kd; \ + struct gcm_context_data ctx; \ + vnet_crypto_op_t *op = ops[i]; \ + \ + kd = (struct gcm_key_data *) imbm->key_data[op->key_index]; \ + IMB_AES##b##_GCM_ENC (m, kd, &ctx, op->dst, op->src, op->len, op->iv, \ + op->aad, op->aad_len, op->tag, op->tag_len); \ + \ + op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; \ + } \ + \ + return n_ops; \ +} \ + \ +static_always_inline u32 \ +ipsecmb_ops_gcm_cipher_dec_##a##_chained (vlib_main_t * vm, \ + vnet_crypto_op_t * ops[], vnet_crypto_op_chunk_t *chunks, u32 n_ops) \ +{ \ + ipsecmb_main_t *imbm = &ipsecmb_main; \ + ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, \ + vm->thread_index); \ + MB_MGR *m = ptd->mgr; \ + vnet_crypto_op_chunk_t *chp; \ + u32 i, j, n_failed = 0; \ + \ + for (i = 0; i < n_ops; i++) \ + { \ + struct gcm_key_data *kd; \ + struct gcm_context_data ctx; \ + vnet_crypto_op_t *op = ops[i]; \ + u8 scratch[64]; \ + \ + kd = (struct gcm_key_data *) imbm->key_data[op->key_index]; \ + ASSERT (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS); \ + IMB_AES##b##_GCM_INIT(m, kd, &ctx, op->iv, op->aad, op->aad_len); \ + chp = chunks + op->chunk_index; \ + for (j = 0; j < op->n_chunks; j++) \ + { \ + IMB_AES##b##_GCM_DEC_UPDATE (m, kd, &ctx, chp->dst, chp->src, \ + chp->len); \ + chp += 1; \ + } \ + IMB_AES##b##_GCM_DEC_FINALIZE(m, kd, &ctx, scratch, op->tag_len); \ + \ + if ((memcmp (op->tag, scratch, op->tag_len))) \ + { \ + op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; \ + n_failed++; \ + } \ + else \ + op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; \ + } \ + \ + return n_ops - n_failed; \ +} \ \ static_always_inline u32 \ ipsecmb_ops_gcm_cipher_dec_##a (vlib_main_t * vm, vnet_crypto_op_t * ops[], \ - u32 n_ops) \ -{ return ipsecmb_ops_gcm_cipher_inline (vm, ops, n_ops, b, DECRYPT); } \ + u32 n_ops) \ +{ \ + ipsecmb_main_t *imbm = &ipsecmb_main; \ + ipsecmb_per_thread_data_t *ptd = vec_elt_at_index (imbm->per_thread_data, \ + vm->thread_index); \ + MB_MGR *m = ptd->mgr; \ + u32 i, n_failed = 0; \ + \ + for (i = 0; i < n_ops; i++) \ + { \ + struct gcm_key_data *kd; \ + struct gcm_context_data ctx; \ + vnet_crypto_op_t *op = ops[i]; \ + u8 scratch[64]; \ + \ + kd = (struct gcm_key_data *) imbm->key_data[op->key_index]; \ + IMB_AES##b##_GCM_DEC (m, kd, &ctx, op->dst, op->src, op->len, op->iv, \ + op->aad, op->aad_len, scratch, op->tag_len); \ + \ + if ((memcmp (op->tag, scratch, op->tag_len))) \ + { \ + op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; \ + n_failed++; \ + } \ + else \ + op->status = VNET_CRYPTO_OP_STATUS_COMPLETED; \ + } \ + \ + return n_ops - n_failed; \ +} foreach_ipsecmb_gcm_cipher_op; #undef _ @@ -402,9 +443,7 @@ crypto_ipsecmb_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop, if (imbm->key_data[idx] == 0) return; - clib_memset_u8 (imbm->key_data[idx], 0, - clib_mem_size (imbm->key_data[idx])); - clib_mem_free (imbm->key_data[idx]); + clib_mem_free_s (imbm->key_data[idx]); imbm->key_data[idx] = 0; return; } @@ -416,9 +455,7 @@ crypto_ipsecmb_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop, if (kop == VNET_CRYPTO_KEY_OP_MODIFY && imbm->key_data[idx]) { - clib_memset_u8 (imbm->key_data[idx], 0, - clib_mem_size (imbm->key_data[idx])); - clib_mem_free (imbm->key_data[idx]); + clib_mem_free_s (imbm->key_data[idx]); } kd = imbm->key_data[idx] = clib_mem_alloc_aligned (ad->data_size, @@ -475,9 +512,6 @@ crypto_ipsecmb_init (vlib_main_t * vm) u32 eidx; u8 *name; - if ((error = vlib_call_init_function (vm, vnet_crypto_init))) - return error; - if (!clib_cpu_supports_aes ()) return 0; @@ -536,6 +570,12 @@ crypto_ipsecmb_init (vlib_main_t * vm) ipsecmb_ops_gcm_cipher_enc_##a); \ vnet_crypto_register_ops_handler (vm, eidx, VNET_CRYPTO_OP_##a##_DEC, \ ipsecmb_ops_gcm_cipher_dec_##a); \ + vnet_crypto_register_chained_ops_handler \ + (vm, eidx, VNET_CRYPTO_OP_##a##_ENC, \ + ipsecmb_ops_gcm_cipher_enc_##a##_chained); \ + vnet_crypto_register_chained_ops_handler \ + (vm, eidx, VNET_CRYPTO_OP_##a##_DEC, \ + ipsecmb_ops_gcm_cipher_dec_##a##_chained); \ ad = imbm->alg_data + VNET_CRYPTO_ALG_##a; \ ad->data_size = sizeof (struct gcm_key_data); \ ad->aes_gcm_pre = m->gcm##b##_pre; \ @@ -547,7 +587,12 @@ crypto_ipsecmb_init (vlib_main_t * vm) return (NULL); } -VLIB_INIT_FUNCTION (crypto_ipsecmb_init); +/* *INDENT-OFF* */ +VLIB_INIT_FUNCTION (crypto_ipsecmb_init) = +{ + .runs_after = VLIB_INITS ("vnet_crypto_init"), +}; +/* *INDENT-ON* */ /* *INDENT-OFF* */ VLIB_PLUGIN_REGISTER () =