+ crypto_len = esp_decrypt_chain_crypto (vm, ptd, pd, pd2, sa0, b, icv_sz,
+ payload,
+ len - pd->iv_sz + pd->icv_sz,
+ &tag, 0);
+ }
+
+ *async_pd = *pd;
+ *async_pd2 = *pd2;
+ pd->protect_index = current_protect_index;
+
+ /* for AEAD integ_len - crypto_len will be negative, it is ok since it
+ * is ignored by the engine. */
+ vnet_crypto_async_add_to_frame (
+ vm, f, key_index, crypto_len, integ_len - crypto_len, crypto_start_offset,
+ integ_start_offset, bi, async_next, iv, tag, aad, flags);
+
+ return (ESP_DECRYPT_ERROR_RX_PKTS);
+}
+
+static_always_inline void
+esp_decrypt_post_crypto (vlib_main_t * vm, vlib_node_runtime_t * node,
+ esp_decrypt_packet_data_t * pd,
+ esp_decrypt_packet_data2_t * pd2, vlib_buffer_t * b,
+ u16 * next, int is_ip6, int is_tun, int is_async)
+{
+ ipsec_sa_t *sa0 = ipsec_sa_get (pd->sa_index);
+ vlib_buffer_t *lb = b;
+ const u8 esp_sz = sizeof (esp_header_t);
+ const u8 tun_flags = IPSEC_SA_FLAG_IS_TUNNEL | IPSEC_SA_FLAG_IS_TUNNEL_V6;
+ u8 pad_length = 0, next_header = 0;
+ u16 icv_sz;
+
+ /*
+ * redo the anti-reply check
+ * in this frame say we have sequence numbers, s, s+1, s+1, s+1
+ * and s and s+1 are in the window. When we did the anti-replay
+ * check above we did so against the state of the window (W),
+ * after packet s-1. So each of the packets in the sequence will be
+ * accepted.
+ * This time s will be cheked against Ws-1, s+1 chceked against Ws
+ * (i.e. the window state is updated/advnaced)
+ * so this time the successive s+! packet will be dropped.
+ * This is a consequence of batching the decrypts. If the
+ * check-dcrypt-advance process was done for each packet it would
+ * be fine. But we batch the decrypts because it's much more efficient
+ * to do so in SW and if we offload to HW and the process is async.
+ *
+ * You're probably thinking, but this means an attacker can send the
+ * above sequence and cause VPP to perform decrpyts that will fail,
+ * and that's true. But if the attacker can determine s (a valid
+ * sequence number in the window) which is non-trivial, it can generate
+ * a sequence s, s+1, s+2, s+3, ... s+n and nothing will prevent any
+ * implementation, sequential or batching, from decrypting these.
+ */
+ if (ipsec_sa_anti_replay_check (sa0, pd->seq))
+ {
+ b->error = node->errors[ESP_DECRYPT_ERROR_REPLAY];
+ next[0] = ESP_DECRYPT_NEXT_DROP;
+ return;
+ }
+
+ ipsec_sa_anti_replay_advance (sa0, pd->seq);
+
+ if (pd->is_chain)
+ {
+ lb = pd2->lb;
+ icv_sz = pd2->icv_removed ? 0 : pd->icv_sz;
+ if (pd2->free_buffer_index)
+ {
+ vlib_buffer_free_one (vm, pd2->free_buffer_index);
+ lb->next_buffer = 0;
+ }
+ if (lb->current_length < sizeof (esp_footer_t) + icv_sz)
+ {
+ /* esp footer is either splitted in two buffers or in the before
+ * last buffer */
+
+ vlib_buffer_t *before_last = b, *bp = b;
+ while (bp->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ before_last = bp;
+ bp = vlib_get_buffer (vm, bp->next_buffer);
+ }
+ u8 *bt = vlib_buffer_get_tail (before_last);
+
+ if (lb->current_length == icv_sz)
+ {
+ esp_footer_t *f = (esp_footer_t *) (bt - sizeof (*f));
+ pad_length = f->pad_length;
+ next_header = f->next_header;
+ }
+ else
+ {
+ pad_length = (bt - 1)[0];
+ next_header = ((u8 *) vlib_buffer_get_current (lb))[0];
+ }
+ }
+ else
+ {
+ esp_footer_t *f =
+ (esp_footer_t *) (lb->data + lb->current_data +
+ lb->current_length - sizeof (esp_footer_t) -
+ icv_sz);
+ pad_length = f->pad_length;
+ next_header = f->next_header;
+ }
+ }
+ else
+ {
+ icv_sz = pd->icv_sz;
+ esp_footer_t *f =
+ (esp_footer_t *) (lb->data + lb->current_data + lb->current_length -
+ sizeof (esp_footer_t) - icv_sz);
+ pad_length = f->pad_length;
+ next_header = f->next_header;
+ }
+
+ u16 adv = pd->iv_sz + esp_sz;
+ u16 tail = sizeof (esp_footer_t) + pad_length + icv_sz;
+ u16 tail_orig = sizeof (esp_footer_t) + pad_length + pd->icv_sz;
+ b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
+
+ if ((pd->flags & tun_flags) == 0 && !is_tun) /* transport mode */
+ {
+ u8 udp_sz = (is_ip6 == 0 && pd->flags & IPSEC_SA_FLAG_UDP_ENCAP) ?
+ sizeof (udp_header_t) : 0;
+ u16 ip_hdr_sz = pd->hdr_sz - udp_sz;
+ u8 *old_ip = b->data + pd->current_data - ip_hdr_sz - udp_sz;
+ u8 *ip = old_ip + adv + udp_sz;
+
+ if (is_ip6 && ip_hdr_sz > 64)
+ memmove (ip, old_ip, ip_hdr_sz);
+ else
+ clib_memcpy_le64 (ip, old_ip, ip_hdr_sz);
+
+ b->current_data = pd->current_data + adv - ip_hdr_sz;
+ b->current_length += ip_hdr_sz - adv;
+ esp_remove_tail (vm, b, lb, tail);
+
+ if (is_ip6)
+ {
+ ip6_header_t *ip6 = (ip6_header_t *) ip;
+ u16 len = clib_net_to_host_u16 (ip6->payload_length);
+ len -= adv + tail_orig;
+ ip6->payload_length = clib_host_to_net_u16 (len);
+ ip6->protocol = next_header;
+ next[0] = ESP_DECRYPT_NEXT_IP6_INPUT;
+ }
+ else
+ {
+ ip4_header_t *ip4 = (ip4_header_t *) ip;
+ ip_csum_t sum = ip4->checksum;
+ u16 len = clib_net_to_host_u16 (ip4->length);
+ len = clib_host_to_net_u16 (len - adv - tail_orig - udp_sz);
+ sum = ip_csum_update (sum, ip4->protocol, next_header,
+ ip4_header_t, protocol);
+ sum = ip_csum_update (sum, ip4->length, len, ip4_header_t, length);
+ ip4->checksum = ip_csum_fold (sum);
+ ip4->protocol = next_header;
+ ip4->length = len;
+ next[0] = ESP_DECRYPT_NEXT_IP4_INPUT;
+ }
+ }
+ else
+ {
+ if (PREDICT_TRUE (next_header == IP_PROTOCOL_IP_IN_IP))
+ {
+ next[0] = ESP_DECRYPT_NEXT_IP4_INPUT;
+ b->current_data = pd->current_data + adv;
+ b->current_length = pd->current_length - adv;
+ esp_remove_tail (vm, b, lb, tail);
+ }
+ else if (next_header == IP_PROTOCOL_IPV6)
+ {
+ next[0] = ESP_DECRYPT_NEXT_IP6_INPUT;
+ b->current_data = pd->current_data + adv;
+ b->current_length = pd->current_length - adv;
+ esp_remove_tail (vm, b, lb, tail);
+ }
+ else if (next_header == IP_PROTOCOL_MPLS_IN_IP)
+ {
+ next[0] = ESP_DECRYPT_NEXT_MPLS_INPUT;
+ b->current_data = pd->current_data + adv;
+ b->current_length = pd->current_length - adv;
+ esp_remove_tail (vm, b, lb, tail);
+ }
+ else
+ {
+ if (is_tun && next_header == IP_PROTOCOL_GRE)
+ {
+ gre_header_t *gre;
+
+ b->current_data = pd->current_data + adv;
+ b->current_length = pd->current_length - adv - tail;
+
+ gre = vlib_buffer_get_current (b);
+
+ vlib_buffer_advance (b, sizeof (*gre));
+
+ switch (clib_net_to_host_u16 (gre->protocol))