2 * esp_decrypt.c : IPSec ESP Decrypt node using DPDK Cryptodev
4 * Copyright (c) 2016 Intel and/or its affiliates.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at:
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
18 #include <vnet/vnet.h>
19 #include <vnet/api_errno.h>
20 #include <vnet/ip/ip.h>
22 #include <vnet/ipsec/ipsec.h>
23 #include <vnet/devices/dpdk/ipsec/ipsec.h>
24 #include <vnet/devices/dpdk/ipsec/esp.h>
26 #define foreach_esp_decrypt_next \
27 _(DROP, "error-drop") \
28 _(IP4_INPUT, "ip4-input") \
29 _(IP6_INPUT, "ip6-input")
31 #define _(v, s) ESP_DECRYPT_NEXT_##v,
33 foreach_esp_decrypt_next
38 #define foreach_esp_decrypt_error \
39 _(RX_PKTS, "ESP pkts received") \
40 _(DECRYPTION_FAILED, "ESP decryption failed") \
41 _(REPLAY, "SA replayed packet") \
42 _(NOT_IP, "Not IP packet (dropped)") \
43 _(ENQ_FAIL, "Enqueue failed (buffer full)") \
44 _(NO_CRYPTODEV, "Cryptodev not configured") \
45 _(BAD_LEN, "Invalid ciphertext length") \
46 _(UNSUPPORTED, "Cipher/Auth not supported")
50 #define _(sym,str) ESP_DECRYPT_ERROR_##sym,
51 foreach_esp_decrypt_error
54 } esp_decrypt_error_t;
56 static char * esp_decrypt_error_strings[] = {
57 #define _(sym,string) string,
58 foreach_esp_decrypt_error
62 vlib_node_registration_t dpdk_esp_decrypt_node;
65 ipsec_crypto_alg_t crypto_alg;
66 ipsec_integ_alg_t integ_alg;
67 } esp_decrypt_trace_t;
69 /* packet trace format function */
70 static u8 * format_esp_decrypt_trace (u8 * s, va_list * args)
72 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
73 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
74 esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *);
76 s = format (s, "esp: crypto %U integrity %U",
77 format_ipsec_crypto_alg, t->crypto_alg,
78 format_ipsec_integ_alg, t->integ_alg);
83 dpdk_esp_decrypt_node_fn (vlib_main_t * vm,
84 vlib_node_runtime_t * node,
85 vlib_frame_t * from_frame)
87 u32 n_left_from, *from, *to_next, next_index;
88 ipsec_main_t *im = &ipsec_main;
89 u32 cpu_index = os_get_cpu_number();
90 dpdk_crypto_main_t * dcm = &dpdk_crypto_main;
91 dpdk_esp_main_t * em = &dpdk_esp_main;
94 from = vlib_frame_vector_args (from_frame);
95 n_left_from = from_frame->n_vectors;
97 if (PREDICT_FALSE(!dcm->workers_main))
99 vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
100 ESP_DECRYPT_ERROR_NO_CRYPTODEV, n_left_from);
101 vlib_buffer_free(vm, from, n_left_from);
105 crypto_worker_main_t *cwm = vec_elt_at_index(dcm->workers_main, cpu_index);
106 u32 n_qps = vec_len(cwm->qp_data);
107 struct rte_crypto_op ** cops_to_enq[n_qps];
108 u32 n_cop_qp[n_qps], * bi_to_enq[n_qps];
110 for (i = 0; i < n_qps; i++)
112 bi_to_enq[i] = cwm->qp_data[i].bi;
113 cops_to_enq[i] = cwm->qp_data[i].cops;
116 memset(n_cop_qp, 0, n_qps * sizeof(u32));
120 next_index = ESP_DECRYPT_NEXT_DROP;
122 while (n_left_from > 0)
126 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
128 while (n_left_from > 0 && n_left_to_next > 0)
130 u32 bi0, sa_index0 = ~0, seq, icv_size;
134 struct rte_mbuf * mb0 = 0;
135 const int BLOCK_SIZE = 16;
136 const int IV_SIZE = 16;
137 crypto_sa_session_t * sa_sess;
140 struct rte_crypto_op * cop = 0;
146 b0 = vlib_get_buffer (vm, bi0);
147 esp0 = vlib_buffer_get_current (b0);
149 sa_index0 = vnet_buffer(b0)->ipsec.sad_index;
150 sa0 = pool_elt_at_index (im->sad, sa_index0);
152 seq = clib_host_to_net_u32(esp0->seq);
154 /* anti-replay check */
155 if (sa0->use_anti_replay)
159 if (PREDICT_TRUE(sa0->use_esn))
160 rv = esp_replay_check_esn(sa0, seq);
162 rv = esp_replay_check(sa0, seq);
164 if (PREDICT_FALSE(rv))
166 clib_warning ("anti-replay SPI %u seq %u", sa0->spi, seq);
167 vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
168 ESP_DECRYPT_ERROR_REPLAY, 1);
176 if (PREDICT_FALSE(sa0->integ_alg == IPSEC_INTEG_ALG_NONE) ||
177 PREDICT_FALSE(sa0->crypto_alg == IPSEC_CRYPTO_ALG_NONE))
179 clib_warning ("SPI %u : only cipher + auth supported", sa0->spi);
180 vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
181 ESP_DECRYPT_ERROR_UNSUPPORTED, 1);
188 sa_sess = pool_elt_at_index(cwm->sa_sess_d[0], sa_index0);
190 if (PREDICT_FALSE(!sa_sess->sess))
192 int ret = create_sym_sess(sa0, sa_sess, 0);
196 sess = sa_sess->sess;
197 qp_index = sa_sess->qp_index;
199 ASSERT (vec_len (vec_elt (cwm->qp_data, qp_index).free_cops) > 0);
200 cop = vec_pop (vec_elt (cwm->qp_data, qp_index).free_cops);
201 ASSERT (cop->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED);
203 cops_to_enq[qp_index][0] = cop;
204 cops_to_enq[qp_index] += 1;
205 n_cop_qp[qp_index] += 1;
206 bi_to_enq[qp_index][0] = bi0;
207 bi_to_enq[qp_index] += 1;
209 rte_crypto_op_attach_sym_session(cop, sess);
211 icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size;
213 /* Convert vlib buffer to mbuf */
214 mb0 = rte_mbuf_from_vlib_buffer(b0);
215 mb0->data_len = b0->current_length;
216 mb0->pkt_len = b0->current_length;
217 mb0->data_off = RTE_PKTMBUF_HEADROOM + b0->current_data;
219 /* Outer IP header has already been stripped */
220 u16 payload_len = rte_pktmbuf_pkt_len(mb0) - sizeof (esp_header_t) -
223 if ((payload_len & (BLOCK_SIZE - 1)) || (payload_len <= 0))
225 clib_warning ("payload %u not multiple of %d\n",
226 payload_len, BLOCK_SIZE);
227 vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
228 ESP_DECRYPT_ERROR_BAD_LEN, 1);
229 vec_add (vec_elt (cwm->qp_data, qp_index).free_cops, &cop, 1);
230 bi_to_enq[qp_index] -= 1;
231 cops_to_enq[qp_index] -= 1;
232 n_cop_qp[qp_index] -= 1;
239 struct rte_crypto_sym_op *sym_cop = (struct rte_crypto_sym_op *)(cop + 1);
241 sym_cop->m_src = mb0;
242 sym_cop->cipher.data.offset = sizeof (esp_header_t) + IV_SIZE;
243 sym_cop->cipher.data.length = payload_len;
245 sym_cop->cipher.iv.data =
246 rte_pktmbuf_mtod_offset(mb0, void*, sizeof (esp_header_t));
247 sym_cop->cipher.iv.phys_addr =
248 rte_pktmbuf_mtophys_offset(mb0, sizeof (esp_header_t));
249 sym_cop->cipher.iv.length = IV_SIZE;
253 dpdk_cop_priv_t* priv = (dpdk_cop_priv_t*) (sym_cop + 1);
255 rte_pktmbuf_mtod_offset(mb0, u8*, sizeof(esp_header_t) + IV_SIZE +
258 memcpy (priv->icv, payload_end, icv_size);
259 *((u32*) payload_end) = sa0->seq_hi;
260 sym_cop->auth.data.offset = 0;
261 sym_cop->auth.data.length =
262 sizeof(esp_header_t) + IV_SIZE + payload_len + sizeof(sa0->seq_hi);
263 sym_cop->auth.digest.data = priv->icv;
264 sym_cop->auth.digest.phys_addr =
265 cop->phys_addr + (uintptr_t) priv->icv - (uintptr_t) cop;
266 sym_cop->auth.digest.length = icv_size;
270 sym_cop->auth.data.offset = 0;
271 sym_cop->auth.data.length = sizeof(esp_header_t) +
272 IV_SIZE + payload_len;
274 sym_cop->auth.digest.data =
275 rte_pktmbuf_mtod_offset(mb0, void*,
276 rte_pktmbuf_pkt_len(mb0) - icv_size);
277 sym_cop->auth.digest.phys_addr =
278 rte_pktmbuf_mtophys_offset(mb0,
279 rte_pktmbuf_pkt_len(mb0) - icv_size);
280 sym_cop->auth.digest.length = icv_size;
284 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
286 esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
287 tr->crypto_alg = sa0->crypto_alg;
288 tr->integ_alg = sa0->integ_alg;
291 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
293 vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
294 ESP_DECRYPT_ERROR_RX_PKTS,
295 from_frame->n_vectors);
296 crypto_qp_data_t *qpd;
298 vec_foreach_index (i, cwm->qp_data)
302 qpd = vec_elt_at_index(cwm->qp_data, i);
303 enq = rte_cryptodev_enqueue_burst(qpd->dev_id, qpd->qp_id,
304 qpd->cops, n_cop_qp[i]);
305 qpd->inflights += enq;
307 if (PREDICT_FALSE(enq < n_cop_qp[i]))
309 crypto_free_cop (qpd, &qpd->cops[enq], n_cop_qp[i] - enq);
310 vlib_buffer_free (vm, &qpd->bi[enq], n_cop_qp[i] - enq);
312 vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
313 ESP_DECRYPT_ERROR_ENQ_FAIL,
319 return from_frame->n_vectors;
322 VLIB_REGISTER_NODE (dpdk_esp_decrypt_node) = {
323 .function = dpdk_esp_decrypt_node_fn,
324 .name = "dpdk-esp-decrypt",
325 .vector_size = sizeof (u32),
326 .format_trace = format_esp_decrypt_trace,
327 .type = VLIB_NODE_TYPE_INTERNAL,
329 .n_errors = ARRAY_LEN(esp_decrypt_error_strings),
330 .error_strings = esp_decrypt_error_strings,
332 .n_next_nodes = ESP_DECRYPT_N_NEXT,
334 #define _(s,n) [ESP_DECRYPT_NEXT_##s] = n,
335 foreach_esp_decrypt_next
340 VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_node, dpdk_esp_decrypt_node_fn)
346 #define foreach_esp_decrypt_post_error \
347 _(PKTS, "ESP post pkts")
350 #define _(sym,str) ESP_DECRYPT_POST_ERROR_##sym,
351 foreach_esp_decrypt_post_error
353 ESP_DECRYPT_POST_N_ERROR,
354 } esp_decrypt_post_error_t;
356 static char * esp_decrypt_post_error_strings[] = {
357 #define _(sym,string) string,
358 foreach_esp_decrypt_post_error
362 vlib_node_registration_t dpdk_esp_decrypt_post_node;
364 static u8 * format_esp_decrypt_post_trace (u8 * s, va_list * args)
370 dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm,
371 vlib_node_runtime_t * node,
372 vlib_frame_t * from_frame)
374 u32 n_left_from, *from, *to_next = 0, next_index;
377 ipsec_main_t *im = &ipsec_main;
378 dpdk_esp_main_t *em = &dpdk_esp_main;
380 from = vlib_frame_vector_args (from_frame);
381 n_left_from = from_frame->n_vectors;
383 next_index = node->cached_next_index;
385 while (n_left_from > 0)
389 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
391 while (n_left_from > 0 && n_left_to_next > 0)
394 const u32 IV_SIZE = 16;
395 u32 bi0, next0, icv_size;
396 vlib_buffer_t * b0 = 0;
397 ip4_header_t *ih4 = 0, *oh4 = 0;
398 ip6_header_t *ih6 = 0, *oh6 = 0;
400 u8 transport_ip6 = 0;
402 next0 = ESP_DECRYPT_NEXT_DROP;
409 b0 = vlib_get_buffer (vm, bi0);
411 sa_index0 = vnet_buffer(b0)->ipsec.sad_index;
412 sa0 = pool_elt_at_index (im->sad, sa_index0);
417 icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size;
419 if (sa0->use_anti_replay)
421 esp_header_t * esp0 = vlib_buffer_get_current (b0);
423 seq = clib_host_to_net_u32(esp0->seq);
424 if (PREDICT_TRUE(sa0->use_esn))
425 esp_replay_advance_esn(sa0, seq);
427 esp_replay_advance(sa0, seq);
430 ih4 = (ip4_header_t *) (b0->data + sizeof(ethernet_header_t));
431 vlib_buffer_advance (b0, sizeof (esp_header_t) + IV_SIZE);
433 b0->current_length -= (icv_size + 2);
434 b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
435 f0 = (esp_footer_t *) ((u8 *) vlib_buffer_get_current (b0) +
437 b0->current_length -= f0->pad_length;
440 if (PREDICT_FALSE(!sa0->is_tunnel && !sa0->is_tunnel_ip6))
444 if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0) != 0x40))
446 if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0) == 0x60))
450 clib_warning("next header: 0x%x", f0->next_header);
451 vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
452 ESP_DECRYPT_ERROR_NOT_IP, 1);
458 if (PREDICT_TRUE (tunnel_mode))
460 if (PREDICT_TRUE(f0->next_header == IP_PROTOCOL_IP_IN_IP))
461 next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
462 else if (f0->next_header == IP_PROTOCOL_IPV6)
463 next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
466 clib_warning("next header: 0x%x", f0->next_header);
467 vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index,
468 ESP_DECRYPT_ERROR_DECRYPTION_FAILED,
476 if (PREDICT_FALSE(transport_ip6))
478 next0 = ESP_DECRYPT_NEXT_IP6_INPUT;
479 ih6 = (ip6_header_t *) (b0->data + sizeof(ethernet_header_t));
480 vlib_buffer_advance (b0, -sizeof(ip6_header_t));
481 oh6 = vlib_buffer_get_current (b0);
482 memmove(oh6, ih6, sizeof(ip6_header_t));
484 oh6->protocol = f0->next_header;
485 oh6->payload_length =
486 clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0) -
487 sizeof (ip6_header_t));
491 next0 = ESP_DECRYPT_NEXT_IP4_INPUT;
492 vlib_buffer_advance (b0, -sizeof(ip4_header_t));
493 oh4 = vlib_buffer_get_current (b0);
495 oh4->ip_version_and_header_length = 0x45;
497 oh4->fragment_id = 0;
498 oh4->flags_and_fragment_offset = 0;
500 oh4->protocol = f0->next_header;
501 oh4->src_address.as_u32 = ih4->src_address.as_u32;
502 oh4->dst_address.as_u32 = ih4->dst_address.as_u32;
504 clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0));
505 oh4->checksum = ip4_header_checksum (oh4);
509 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32)~0;
512 if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
514 esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
515 tr->crypto_alg = sa0->crypto_alg;
516 tr->integ_alg = sa0->integ_alg;
519 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
520 to_next, n_left_to_next, bi0, next0);
522 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
524 vlib_node_increment_counter (vm, dpdk_esp_decrypt_post_node.index,
525 ESP_DECRYPT_POST_ERROR_PKTS,
526 from_frame->n_vectors);
528 return from_frame->n_vectors;
531 VLIB_REGISTER_NODE (dpdk_esp_decrypt_post_node) = {
532 .function = dpdk_esp_decrypt_post_node_fn,
533 .name = "dpdk-esp-decrypt-post",
534 .vector_size = sizeof (u32),
535 .format_trace = format_esp_decrypt_post_trace,
536 .type = VLIB_NODE_TYPE_INTERNAL,
538 .n_errors = ARRAY_LEN(esp_decrypt_post_error_strings),
539 .error_strings = esp_decrypt_post_error_strings,
541 .n_next_nodes = ESP_DECRYPT_N_NEXT,
543 #define _(s,n) [ESP_DECRYPT_NEXT_##s] = n,
544 foreach_esp_decrypt_next
549 VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_post_node, dpdk_esp_decrypt_post_node_fn)