+static_always_inline void
+esp_process_ops (vlib_main_t * vm, vlib_node_runtime_t * node,
+ vnet_crypto_op_t * ops, vlib_buffer_t * b[], u16 * nexts,
+ int e)
+{
+ vnet_crypto_op_t *op = ops;
+ u32 n_fail, n_ops = vec_len (ops);
+
+ if (n_ops == 0)
+ return;
+
+ n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops);
+
+ while (n_fail)
+ {
+ ASSERT (op - ops < n_ops);
+ if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
+ {
+ u32 err, bi = op->user_data;
+ if (op->status == VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC)
+ err = e;
+ else
+ err = ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR;
+ b[bi]->error = node->errors[err];
+ nexts[bi] = ESP_DECRYPT_NEXT_DROP;
+ n_fail--;
+ }
+ op++;
+ }
+}
+
+static_always_inline void
+esp_process_chained_ops (vlib_main_t * vm, vlib_node_runtime_t * node,
+ vnet_crypto_op_t * ops, vlib_buffer_t * b[],
+ u16 * nexts, vnet_crypto_op_chunk_t * chunks, int e)
+{
+
+ vnet_crypto_op_t *op = ops;
+ u32 n_fail, n_ops = vec_len (ops);
+
+ if (n_ops == 0)
+ return;
+
+ n_fail = n_ops - vnet_crypto_process_chained_ops (vm, op, chunks, n_ops);
+
+ while (n_fail)
+ {
+ ASSERT (op - ops < n_ops);
+ if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
+ {
+ u32 err, bi = op->user_data;
+ if (op->status == VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC)
+ err = e;
+ else
+ err = ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR;
+ b[bi]->error = node->errors[err];
+ nexts[bi] = ESP_DECRYPT_NEXT_DROP;
+ n_fail--;
+ }
+ op++;
+ }
+}
+
+always_inline void
+esp_remove_tail (vlib_main_t * vm, vlib_buffer_t * b, vlib_buffer_t * last,
+ u16 tail)
+{
+ vlib_buffer_t *before_last = b;
+
+ if (last->current_length > tail)
+ {
+ last->current_length -= tail;
+ return;
+ }
+ ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT);
+
+ while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ before_last = b;
+ b = vlib_get_buffer (vm, b->next_buffer);
+ }
+ before_last->current_length -= tail - last->current_length;
+ vlib_buffer_free_one (vm, before_last->next_buffer);
+ before_last->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
+}
+
+/* ICV is splitted in last two buffers so move it to the last buffer and
+ return pointer to it */
+static_always_inline u8 *
+esp_move_icv (vlib_main_t * vm, vlib_buffer_t * first,
+ esp_decrypt_packet_data_t * pd, u16 icv_sz, u16 * dif)
+{
+ vlib_buffer_t *before_last, *bp;
+ u16 last_sz = pd->lb->current_length;
+ u16 first_sz = icv_sz - last_sz;
+
+ bp = before_last = first;
+ while (bp->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ before_last = bp;
+ bp = vlib_get_buffer (vm, bp->next_buffer);
+ }
+
+ u8 *lb_curr = vlib_buffer_get_current (pd->lb);
+ memmove (lb_curr + first_sz, lb_curr, last_sz);
+ clib_memcpy_fast (lb_curr, vlib_buffer_get_tail (before_last) - first_sz,
+ first_sz);
+ before_last->current_length -= first_sz;
+ if (dif)
+ dif[0] = first_sz;
+ pd->lb = before_last;
+ pd->icv_removed = 1;
+ pd->free_buffer_index = before_last->next_buffer;
+ before_last->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
+ return lb_curr;
+}
+
+static_always_inline int
+esp_insert_esn (vlib_main_t * vm, ipsec_sa_t * sa,
+ esp_decrypt_packet_data_t * pd, vnet_crypto_op_t * op,
+ u16 * len, vlib_buffer_t * b, u8 * payload)
+{
+ if (!ipsec_sa_is_set_USE_ESN (sa))
+ return 1;
+
+ /* shift ICV by 4 bytes to insert ESN */
+ u32 seq_hi = clib_host_to_net_u32 (sa->seq_hi);
+ u8 tmp[ESP_MAX_ICV_SIZE], sz = sizeof (sa->seq_hi);
+
+ if (pd->icv_removed)
+ {
+ u16 space_left = vlib_buffer_space_left_at_end (vm, pd->lb);
+ if (space_left >= sz)
+ {
+ clib_memcpy_fast (vlib_buffer_get_tail (pd->lb), &seq_hi, sz);
+ op->len += sz;
+ }
+ else
+ return 0;
+
+ len[0] = b->current_length;
+ }
+ else
+ {
+ clib_memcpy_fast (tmp, payload + len[0], ESP_MAX_ICV_SIZE);
+ clib_memcpy_fast (payload + len[0], &seq_hi, sz);
+ clib_memcpy_fast (payload + len[0] + sz, tmp, ESP_MAX_ICV_SIZE);
+ op->len += sz;
+ op->digest += sz;
+ }
+ return 1;
+}
+
+static_always_inline u8 *
+esp_move_icv_esn (vlib_main_t * vm, vlib_buffer_t * first,
+ esp_decrypt_packet_data_t * pd, u16 icv_sz, ipsec_sa_t * sa,
+ u8 * extra_esn, vnet_crypto_op_t * op)
+{
+ u16 dif = 0;
+ u8 *digest = esp_move_icv (vm, first, pd, icv_sz, &dif);
+ if (dif)
+ op->len -= dif;
+
+ if (ipsec_sa_is_set_USE_ESN (sa))
+ {
+ u8 sz = sizeof (sa->seq_hi);
+ u32 seq_hi = clib_host_to_net_u32 (sa->seq_hi);
+ u16 space_left = vlib_buffer_space_left_at_end (vm, pd->lb);
+
+ if (space_left >= sz)
+ {
+ clib_memcpy_fast (vlib_buffer_get_tail (pd->lb), &seq_hi, sz);
+ op->len += sz;
+ }
+ else
+ {
+ /* no space for ESN at the tail, use the next buffer
+ * (with ICV data) */
+ ASSERT (pd->icv_removed);
+ vlib_buffer_t *tmp = vlib_get_buffer (vm, pd->free_buffer_index);
+ clib_memcpy_fast (vlib_buffer_get_current (tmp) - sz, &seq_hi, sz);
+ extra_esn[0] = 1;
+ }
+ }
+ return digest;
+}
+