2 * esp_encrypt.c : IPSec ESP encrypt node
4 * Copyright (c) 2015 Cisco and/or its affiliates.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at:
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
18 #include <vnet/ipsec/ipsec.h>
19 #include <vnet/ipsec/ipsec_sa.h>
21 #define foreach_ipsec_handoff_error \
22 _(CONGESTION_DROP, "congestion drop")
26 #define _(sym,str) IPSEC_HANDOFF_ERROR_##sym,
27 foreach_ipsec_handoff_error
29 NAT44_HANDOFF_N_ERROR,
30 } ipsec_handoff_error_t;
32 static char *ipsec_handoff_error_strings[] = {
33 #define _(sym,string) string,
34 foreach_ipsec_handoff_error
38 typedef struct ipsec_handoff_trace_t_
40 u32 next_worker_index;
41 } ipsec_handoff_trace_t;
44 format_ipsec_handoff_trace (u8 * s, va_list * args)
46 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
47 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
48 ipsec_handoff_trace_t *t = va_arg (*args, ipsec_handoff_trace_t *);
50 s = format (s, "next-worker %d", t->next_worker_index);
55 /* do worker handoff based on thread_index in NAT HA protcol header */
56 static_always_inline uword
57 ipsec_handoff (vlib_main_t * vm,
58 vlib_node_runtime_t * node,
59 vlib_frame_t * frame, u32 fq_index, bool is_enc)
61 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
62 u16 thread_indices[VLIB_FRAME_SIZE], *ti;
63 u32 n_enq, n_left_from, *from;
67 from = vlib_frame_vector_args (frame);
68 n_left_from = frame->n_vectors;
69 vlib_get_buffers (vm, from, bufs, n_left_from);
74 while (n_left_from >= 4)
76 ipsec_sa_t *sa0, *sa1, *sa2, *sa3;
77 u32 sai0, sai1, sai2, sai3;
79 /* Prefetch next iteration. */
80 if (n_left_from >= 12)
82 vlib_prefetch_buffer_header (b[8], LOAD);
83 vlib_prefetch_buffer_header (b[9], LOAD);
84 vlib_prefetch_buffer_header (b[10], LOAD);
85 vlib_prefetch_buffer_header (b[11], LOAD);
87 vlib_prefetch_buffer_data (b[4], LOAD);
88 vlib_prefetch_buffer_data (b[5], LOAD);
89 vlib_prefetch_buffer_data (b[6], LOAD);
90 vlib_prefetch_buffer_data (b[7], LOAD);
93 sai0 = vnet_buffer (b[0])->ipsec.sad_index;
94 sai1 = vnet_buffer (b[1])->ipsec.sad_index;
95 sai2 = vnet_buffer (b[2])->ipsec.sad_index;
96 sai3 = vnet_buffer (b[3])->ipsec.sad_index;
97 sa0 = pool_elt_at_index (im->sad, sai0);
98 sa1 = pool_elt_at_index (im->sad, sai1);
99 sa2 = pool_elt_at_index (im->sad, sai2);
100 sa3 = pool_elt_at_index (im->sad, sai3);
104 ti[0] = sa0->encrypt_thread_index;
105 ti[1] = sa1->encrypt_thread_index;
106 ti[2] = sa2->encrypt_thread_index;
107 ti[3] = sa3->encrypt_thread_index;
111 ti[0] = sa0->decrypt_thread_index;
112 ti[1] = sa1->decrypt_thread_index;
113 ti[2] = sa2->decrypt_thread_index;
114 ti[3] = sa3->decrypt_thread_index;
117 if (node->flags & VLIB_NODE_FLAG_TRACE)
119 if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
121 ipsec_handoff_trace_t *t =
122 vlib_add_trace (vm, node, b[0], sizeof (*t));
123 t->next_worker_index = ti[0];
125 if (PREDICT_FALSE (b[1]->flags & VLIB_BUFFER_IS_TRACED))
127 ipsec_handoff_trace_t *t =
128 vlib_add_trace (vm, node, b[1], sizeof (*t));
129 t->next_worker_index = ti[1];
131 if (PREDICT_FALSE (b[2]->flags & VLIB_BUFFER_IS_TRACED))
133 ipsec_handoff_trace_t *t =
134 vlib_add_trace (vm, node, b[2], sizeof (*t));
135 t->next_worker_index = ti[2];
137 if (PREDICT_FALSE (b[3]->flags & VLIB_BUFFER_IS_TRACED))
139 ipsec_handoff_trace_t *t =
140 vlib_add_trace (vm, node, b[3], sizeof (*t));
141 t->next_worker_index = ti[3];
149 while (n_left_from > 0)
154 sai0 = vnet_buffer (b[0])->ipsec.sad_index;
155 sa0 = pool_elt_at_index (im->sad, sai0);
158 ti[0] = sa0->encrypt_thread_index;
160 ti[0] = sa0->decrypt_thread_index;
162 if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
164 ipsec_handoff_trace_t *t =
165 vlib_add_trace (vm, node, b[0], sizeof (*t));
166 t->next_worker_index = ti[0];
174 n_enq = vlib_buffer_enqueue_to_thread (vm, fq_index, from,
175 thread_indices, frame->n_vectors, 1);
177 if (n_enq < frame->n_vectors)
178 vlib_node_increment_counter (vm, node->node_index,
179 IPSEC_HANDOFF_ERROR_CONGESTION_DROP,
180 frame->n_vectors - n_enq);
185 VLIB_NODE_FN (esp4_encrypt_handoff) (vlib_main_t * vm,
186 vlib_node_runtime_t * node,
187 vlib_frame_t * from_frame)
189 ipsec_main_t *im = &ipsec_main;
191 return ipsec_handoff (vm, node, from_frame, im->esp4_enc_fq_index, true);
194 VLIB_NODE_FN (esp6_encrypt_handoff) (vlib_main_t * vm,
195 vlib_node_runtime_t * node,
196 vlib_frame_t * from_frame)
198 ipsec_main_t *im = &ipsec_main;
200 return ipsec_handoff (vm, node, from_frame, im->esp6_enc_fq_index, true);
203 VLIB_NODE_FN (esp4_encrypt_tun_handoff) (vlib_main_t * vm,
204 vlib_node_runtime_t * node,
205 vlib_frame_t * from_frame)
207 ipsec_main_t *im = &ipsec_main;
209 return ipsec_handoff (vm, node, from_frame, im->esp4_enc_tun_fq_index,
213 VLIB_NODE_FN (esp6_encrypt_tun_handoff) (vlib_main_t * vm,
214 vlib_node_runtime_t * node,
215 vlib_frame_t * from_frame)
217 ipsec_main_t *im = &ipsec_main;
219 return ipsec_handoff (vm, node, from_frame, im->esp6_enc_tun_fq_index,
223 VLIB_NODE_FN (esp_mpls_encrypt_tun_handoff)
224 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
226 ipsec_main_t *im = &ipsec_main;
228 return ipsec_handoff (vm, node, from_frame, im->esp_mpls_enc_tun_fq_index,
232 VLIB_NODE_FN (esp4_decrypt_handoff) (vlib_main_t * vm,
233 vlib_node_runtime_t * node,
234 vlib_frame_t * from_frame)
236 ipsec_main_t *im = &ipsec_main;
238 return ipsec_handoff (vm, node, from_frame, im->esp4_dec_fq_index, false);
241 VLIB_NODE_FN (esp6_decrypt_handoff) (vlib_main_t * vm,
242 vlib_node_runtime_t * node,
243 vlib_frame_t * from_frame)
245 ipsec_main_t *im = &ipsec_main;
247 return ipsec_handoff (vm, node, from_frame, im->esp6_dec_fq_index, false);
250 VLIB_NODE_FN (esp4_decrypt_tun_handoff) (vlib_main_t * vm,
251 vlib_node_runtime_t * node,
252 vlib_frame_t * from_frame)
254 ipsec_main_t *im = &ipsec_main;
256 return ipsec_handoff (vm, node, from_frame, im->esp4_dec_tun_fq_index,
260 VLIB_NODE_FN (esp6_decrypt_tun_handoff) (vlib_main_t * vm,
261 vlib_node_runtime_t * node,
262 vlib_frame_t * from_frame)
264 ipsec_main_t *im = &ipsec_main;
266 return ipsec_handoff (vm, node, from_frame, im->esp6_dec_tun_fq_index,
270 VLIB_NODE_FN (ah4_encrypt_handoff) (vlib_main_t * vm,
271 vlib_node_runtime_t * node,
272 vlib_frame_t * from_frame)
274 ipsec_main_t *im = &ipsec_main;
276 return ipsec_handoff (vm, node, from_frame, im->ah4_enc_fq_index, true);
279 VLIB_NODE_FN (ah6_encrypt_handoff) (vlib_main_t * vm,
280 vlib_node_runtime_t * node,
281 vlib_frame_t * from_frame)
283 ipsec_main_t *im = &ipsec_main;
285 return ipsec_handoff (vm, node, from_frame, im->ah6_enc_fq_index, true);
288 VLIB_NODE_FN (ah4_decrypt_handoff) (vlib_main_t * vm,
289 vlib_node_runtime_t * node,
290 vlib_frame_t * from_frame)
292 ipsec_main_t *im = &ipsec_main;
294 return ipsec_handoff (vm, node, from_frame, im->ah4_dec_fq_index, false);
297 VLIB_NODE_FN (ah6_decrypt_handoff) (vlib_main_t * vm,
298 vlib_node_runtime_t * node,
299 vlib_frame_t * from_frame)
301 ipsec_main_t *im = &ipsec_main;
303 return ipsec_handoff (vm, node, from_frame, im->ah6_dec_fq_index, false);
307 VLIB_REGISTER_NODE (esp4_encrypt_handoff) = {
308 .name = "esp4-encrypt-handoff",
309 .vector_size = sizeof (u32),
310 .format_trace = format_ipsec_handoff_trace,
311 .type = VLIB_NODE_TYPE_INTERNAL,
312 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
313 .error_strings = ipsec_handoff_error_strings,
319 VLIB_REGISTER_NODE (esp6_encrypt_handoff) = {
320 .name = "esp6-encrypt-handoff",
321 .vector_size = sizeof (u32),
322 .format_trace = format_ipsec_handoff_trace,
323 .type = VLIB_NODE_TYPE_INTERNAL,
324 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
325 .error_strings = ipsec_handoff_error_strings,
331 VLIB_REGISTER_NODE (esp4_encrypt_tun_handoff) = {
332 .name = "esp4-encrypt-tun-handoff",
333 .vector_size = sizeof (u32),
334 .format_trace = format_ipsec_handoff_trace,
335 .type = VLIB_NODE_TYPE_INTERNAL,
336 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
337 .error_strings = ipsec_handoff_error_strings,
343 VLIB_REGISTER_NODE (esp6_encrypt_tun_handoff) = {
344 .name = "esp6-encrypt-tun-handoff",
345 .vector_size = sizeof (u32),
346 .format_trace = format_ipsec_handoff_trace,
347 .type = VLIB_NODE_TYPE_INTERNAL,
348 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
349 .error_strings = ipsec_handoff_error_strings,
355 VLIB_REGISTER_NODE (esp_mpls_encrypt_tun_handoff) = {
356 .name = "esp-mpls-encrypt-tun-handoff",
357 .vector_size = sizeof (u32),
358 .format_trace = format_ipsec_handoff_trace,
359 .type = VLIB_NODE_TYPE_INTERNAL,
360 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
361 .error_strings = ipsec_handoff_error_strings,
367 VLIB_REGISTER_NODE (esp4_decrypt_handoff) = {
368 .name = "esp4-decrypt-handoff",
369 .vector_size = sizeof (u32),
370 .format_trace = format_ipsec_handoff_trace,
371 .type = VLIB_NODE_TYPE_INTERNAL,
372 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
373 .error_strings = ipsec_handoff_error_strings,
379 VLIB_REGISTER_NODE (esp6_decrypt_handoff) = {
380 .name = "esp6-decrypt-handoff",
381 .vector_size = sizeof (u32),
382 .format_trace = format_ipsec_handoff_trace,
383 .type = VLIB_NODE_TYPE_INTERNAL,
384 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
385 .error_strings = ipsec_handoff_error_strings,
391 VLIB_REGISTER_NODE (esp4_decrypt_tun_handoff) = {
392 .name = "esp4-decrypt-tun-handoff",
393 .vector_size = sizeof (u32),
394 .format_trace = format_ipsec_handoff_trace,
395 .type = VLIB_NODE_TYPE_INTERNAL,
396 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
397 .error_strings = ipsec_handoff_error_strings,
403 VLIB_REGISTER_NODE (esp6_decrypt_tun_handoff) = {
404 .name = "esp6-decrypt-tun-handoff",
405 .vector_size = sizeof (u32),
406 .format_trace = format_ipsec_handoff_trace,
407 .type = VLIB_NODE_TYPE_INTERNAL,
408 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
409 .error_strings = ipsec_handoff_error_strings,
415 VLIB_REGISTER_NODE (ah4_encrypt_handoff) = {
416 .name = "ah4-encrypt-handoff",
417 .vector_size = sizeof (u32),
418 .format_trace = format_ipsec_handoff_trace,
419 .type = VLIB_NODE_TYPE_INTERNAL,
420 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
421 .error_strings = ipsec_handoff_error_strings,
427 VLIB_REGISTER_NODE (ah6_encrypt_handoff) = {
428 .name = "ah6-encrypt-handoff",
429 .vector_size = sizeof (u32),
430 .format_trace = format_ipsec_handoff_trace,
431 .type = VLIB_NODE_TYPE_INTERNAL,
432 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
433 .error_strings = ipsec_handoff_error_strings,
439 VLIB_REGISTER_NODE (ah4_decrypt_handoff) = {
440 .name = "ah4-decrypt-handoff",
441 .vector_size = sizeof (u32),
442 .format_trace = format_ipsec_handoff_trace,
443 .type = VLIB_NODE_TYPE_INTERNAL,
444 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
445 .error_strings = ipsec_handoff_error_strings,
451 VLIB_REGISTER_NODE (ah6_decrypt_handoff) = {
452 .name = "ah6-decrypt-handoff",
453 .vector_size = sizeof (u32),
454 .format_trace = format_ipsec_handoff_trace,
455 .type = VLIB_NODE_TYPE_INTERNAL,
456 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
457 .error_strings = ipsec_handoff_error_strings,
466 * fd.io coding-style-patch-verification: ON
469 * eval: (c-set-style "gnu")