- struct rte_crypto_sym_op *sym_cop = (struct rte_crypto_sym_op *)(cop + 1);
-
- sym_cop->m_src = mb0;
- sym_cop->cipher.data.offset = sizeof (esp_header_t) + iv_size;
- sym_cop->cipher.data.length = payload_len;
-
- u8 *iv = rte_pktmbuf_mtod_offset(mb0, void*, sizeof (esp_header_t));
- dpdk_cop_priv_t * priv = (dpdk_cop_priv_t *)(sym_cop + 1);
-
- if (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
- {
- dpdk_gcm_cnt_blk *icb = &priv->cb;
- icb->salt = sa0->salt;
- clib_memcpy(icb->iv, iv, 8);
- icb->cnt = clib_host_to_net_u32(1);
- sym_cop->cipher.iv.data = (u8 *)icb;
- sym_cop->cipher.iv.phys_addr = cop->phys_addr +
- (uintptr_t)icb - (uintptr_t)cop;
- sym_cop->cipher.iv.length = 16;
-
- u8 *aad = priv->aad;
- clib_memcpy(aad, iv - sizeof(esp_header_t), 8);
- sym_cop->auth.aad.data = aad;
- sym_cop->auth.aad.phys_addr = cop->phys_addr +
- (uintptr_t)aad - (uintptr_t)cop;
- if (sa0->use_esn)
- {
- *((u32*)&aad[8]) = sa0->seq_hi;
- sym_cop->auth.aad.length = 12;
- }
- else
- {
- sym_cop->auth.aad.length = 8;
- }
-
- sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(mb0, void*,
- rte_pktmbuf_pkt_len(mb0) - icv_size);
- sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mb0,
- rte_pktmbuf_pkt_len(mb0) - icv_size);
- sym_cop->auth.digest.length = icv_size;
-
- }
- else
- {
- sym_cop->cipher.iv.data = rte_pktmbuf_mtod_offset(mb0, void*,
- sizeof (esp_header_t));
- sym_cop->cipher.iv.phys_addr = rte_pktmbuf_mtophys_offset(mb0,
- sizeof (esp_header_t));
- sym_cop->cipher.iv.length = iv_size;
-
- if (sa0->use_esn)
- {
- dpdk_cop_priv_t* priv = (dpdk_cop_priv_t*) (sym_cop + 1);
- u8* payload_end = rte_pktmbuf_mtod_offset(
- mb0, u8*, sizeof(esp_header_t) + iv_size + payload_len);
-
- clib_memcpy (priv->icv, payload_end, icv_size);
- *((u32*) payload_end) = sa0->seq_hi;
- sym_cop->auth.data.offset = 0;
- sym_cop->auth.data.length = sizeof(esp_header_t) + iv_size
- + payload_len + sizeof(sa0->seq_hi);
- sym_cop->auth.digest.data = priv->icv;
- sym_cop->auth.digest.phys_addr = cop->phys_addr
- + (uintptr_t) priv->icv - (uintptr_t) cop;
- sym_cop->auth.digest.length = icv_size;
- }
- else
- {
- sym_cop->auth.data.offset = 0;
- sym_cop->auth.data.length = sizeof(esp_header_t) +
- iv_size + payload_len;
-
- sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(mb0, void*,
- rte_pktmbuf_pkt_len(mb0) - icv_size);
- sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mb0,
- rte_pktmbuf_pkt_len(mb0) - icv_size);
- sym_cop->auth.digest.length = icv_size;
- }
- }
-
-trace:
- if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+ u32 cipher_off, cipher_len;
+ u32 auth_len = 0;
+ u8 *aad = NULL;
+
+ u8 *iv = (u8 *) (esp0 + 1);
+
+ dpdk_gcm_cnt_blk *icb = &priv->cb;
+
+ cipher_off = sizeof (esp_header_t) + iv_size;
+ cipher_len = payload_len;
+
+ u8 *digest = vlib_buffer_get_tail (b0) - trunc_size;
+ u64 digest_paddr =
+ mb0->buf_physaddr + digest - ((u8 *) mb0->buf_addr);
+
+ if (!is_aead && cipher_alg->alg == RTE_CRYPTO_CIPHER_AES_CBC)
+ clib_memcpy_fast (icb, iv, 16);
+ else /* CTR/GCM */
+ {
+ u32 *_iv = (u32 *) iv;
+
+ crypto_set_icb (icb, sa0->salt, _iv[0], _iv[1]);
+ }
+
+ if (is_aead)
+ {
+ aad = priv->aad;
+ u32 *_aad = (u32 *) aad;
+ clib_memcpy_fast (aad, esp0, 8);
+
+ /* _aad[3] should always be 0 */
+ if (PREDICT_FALSE (ipsec_sa_is_set_USE_ESN (sa0)))
+ _aad[2] = clib_host_to_net_u32 (sa0->seq_hi);
+ else
+ _aad[2] = 0;
+ }
+ else