- esp_decrypt_cbc (sa0->crypto_alg,
- esp0->data + IV_SIZE,
- (u8 *) vlib_buffer_get_current (o_b0) +
- ip_hdr_size, BLOCK_SIZE * blocks,
- sa0->crypto_key.data, esp0->data);
-
- o_b0->current_length = (blocks * BLOCK_SIZE) - 2 + ip_hdr_size;
- o_b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
- f0 =
- (esp_footer_t *) ((u8 *) vlib_buffer_get_current (o_b0) +
- o_b0->current_length);
- o_b0->current_length -= f0->pad_length;
-
- /* tunnel mode */
- if (PREDICT_TRUE (tunnel_mode))
- {
- if (PREDICT_TRUE (f0->next_header == IP_PROTOCOL_IP_IN_IP))
- {
- next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
- oh4 = vlib_buffer_get_current (o_b0);
- }
- else if (f0->next_header == IP_PROTOCOL_IPV6)
- next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
- else
- {
- vlib_node_increment_counter (vm, node->node_index,
- ESP_DECRYPT_ERROR_DECRYPTION_FAILED,
- 1);
- o_b0 = 0;
- goto trace;
- }
- }
- /* transport mode */
- else
+ n_left = from_frame->n_vectors;
+ next = nexts;
+ pd = pkt_data;
+ b = bufs;
+
+ while (n_left)
+ {
+ const u8 tun_flags = IPSEC_SA_FLAG_IS_TUNNEL |
+ IPSEC_SA_FLAG_IS_TUNNEL_V6;
+
+ if (n_left >= 2)
+ {
+ void *data = b[1]->data + pd[1].current_data;
+
+ /* buffer metadata */
+ vlib_prefetch_buffer_header (b[1], LOAD);
+
+ /* esp_footer_t */
+ CLIB_PREFETCH (data + pd[1].current_length - pd[1].icv_sz - 2,
+ CLIB_CACHE_LINE_BYTES, LOAD);
+
+ /* packet headers */
+ CLIB_PREFETCH (data - CLIB_CACHE_LINE_BYTES,
+ CLIB_CACHE_LINE_BYTES * 2, LOAD);
+ }
+
+ if (next[0] < ESP_DECRYPT_N_NEXT)
+ goto trace;
+
+ sa0 = vec_elt_at_index (im->sad, pd->sa_index);
+ u8 *payload = b[0]->data + pd->current_data;
+
+ ipsec_sa_anti_replay_advance (sa0, ((esp_header_t *) payload)->seq);
+
+ esp_footer_t *f = (esp_footer_t *) (b[0]->data + pd->current_data +
+ pd->current_length - sizeof (*f) -
+ pd->icv_sz);
+ u16 adv = pd->iv_sz + esp_sz;
+ u16 tail = sizeof (esp_footer_t) + f->pad_length + pd->icv_sz;
+
+ if ((pd->flags & tun_flags) == 0 && !is_tun) /* transport mode */
+ {
+ u8 udp_sz = (is_ip6 == 0 && pd->flags & IPSEC_SA_FLAG_UDP_ENCAP) ?
+ sizeof (udp_header_t) : 0;
+ u16 ip_hdr_sz = pd->hdr_sz - udp_sz;
+ u8 *old_ip = b[0]->data + pd->current_data - ip_hdr_sz - udp_sz;
+ u8 *ip = old_ip + adv + udp_sz;
+
+ if (is_ip6 && ip_hdr_sz > 64)
+ memmove (ip, old_ip, ip_hdr_sz);
+ else
+ clib_memcpy_le64 (ip, old_ip, ip_hdr_sz);
+
+ b[0]->current_data = pd->current_data + adv - ip_hdr_sz;
+ b[0]->current_length = pd->current_length + ip_hdr_sz - tail - adv;
+
+ if (is_ip6)
+ {
+ ip6_header_t *ip6 = (ip6_header_t *) ip;
+ u16 len = clib_net_to_host_u16 (ip6->payload_length);
+ len -= adv + tail;
+ ip6->payload_length = clib_host_to_net_u16 (len);
+ ip6->protocol = f->next_header;
+ next[0] = ESP_DECRYPT_NEXT_IP6_INPUT;
+ }
+ else
+ {
+ ip4_header_t *ip4 = (ip4_header_t *) ip;
+ ip_csum_t sum = ip4->checksum;
+ u16 len = clib_net_to_host_u16 (ip4->length);
+ len = clib_host_to_net_u16 (len - adv - tail - udp_sz);
+ sum = ip_csum_update (sum, ip4->protocol, f->next_header,
+ ip4_header_t, protocol);
+ sum = ip_csum_update (sum, ip4->length, len,
+ ip4_header_t, length);
+ ip4->checksum = ip_csum_fold (sum);
+ ip4->protocol = f->next_header;
+ ip4->length = len;
+ next[0] = ESP_DECRYPT_NEXT_IP4_INPUT;
+ }
+ }
+ else
+ {
+ if (PREDICT_TRUE (f->next_header == IP_PROTOCOL_IP_IN_IP))
+ {
+ next[0] = ESP_DECRYPT_NEXT_IP4_INPUT;
+ b[0]->current_data = pd->current_data + adv;
+ b[0]->current_length = pd->current_length - adv - tail;
+ }
+ else if (f->next_header == IP_PROTOCOL_IPV6)
+ {
+ next[0] = ESP_DECRYPT_NEXT_IP6_INPUT;
+ b[0]->current_data = pd->current_data + adv;
+ b[0]->current_length = pd->current_length - adv - tail;
+ }
+ else
+ {
+ next[0] = ESP_DECRYPT_NEXT_DROP;
+ b[0]->error = node->errors[ESP_DECRYPT_ERROR_DECRYPTION_FAILED];
+ goto trace;
+ }
+ if (is_tun)
+ {
+ if (ipsec_sa_is_set_IS_PROTECT (sa0))