2 * esp_encrypt.c : IPSec ESP encrypt node
4 * Copyright (c) 2015 Cisco and/or its affiliates.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at:
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
18 #include <vnet/ipsec/ipsec.h>
19 #include <vnet/ipsec/ipsec_sa.h>
21 #define foreach_ipsec_handoff_error \
22 _(CONGESTION_DROP, "congestion drop")
26 #define _(sym,str) IPSEC_HANDOFF_ERROR_##sym,
27 foreach_ipsec_handoff_error
29 NAT44_HANDOFF_N_ERROR,
30 } ipsec_handoff_error_t;
32 static char *ipsec_handoff_error_strings[] = {
33 #define _(sym,string) string,
34 foreach_ipsec_handoff_error
38 typedef struct ipsec_handoff_trace_t_
40 u32 next_worker_index;
41 } ipsec_handoff_trace_t;
44 format_ipsec_handoff_trace (u8 * s, va_list * args)
46 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
47 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
48 ipsec_handoff_trace_t *t = va_arg (*args, ipsec_handoff_trace_t *);
50 s = format (s, "next-worker %d", t->next_worker_index);
55 /* do worker handoff based on thread_index in NAT HA protcol header */
56 static_always_inline uword
57 ipsec_handoff (vlib_main_t * vm,
58 vlib_node_runtime_t * node,
59 vlib_frame_t * frame, u32 fq_index, bool is_enc)
61 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
62 u16 thread_indices[VLIB_FRAME_SIZE], *ti;
63 u32 n_enq, n_left_from, *from;
67 from = vlib_frame_vector_args (frame);
68 n_left_from = frame->n_vectors;
69 vlib_get_buffers (vm, from, bufs, n_left_from);
74 while (n_left_from >= 4)
76 ipsec_sa_t *sa0, *sa1, *sa2, *sa3;
77 u32 sai0, sai1, sai2, sai3;
79 /* Prefetch next iteration. */
80 if (n_left_from >= 12)
82 vlib_prefetch_buffer_header (b[8], LOAD);
83 vlib_prefetch_buffer_header (b[9], LOAD);
84 vlib_prefetch_buffer_header (b[10], LOAD);
85 vlib_prefetch_buffer_header (b[11], LOAD);
87 vlib_prefetch_buffer_data (b[4], LOAD);
88 vlib_prefetch_buffer_data (b[5], LOAD);
89 vlib_prefetch_buffer_data (b[6], LOAD);
90 vlib_prefetch_buffer_data (b[7], LOAD);
93 sai0 = vnet_buffer (b[0])->ipsec.sad_index;
94 sai1 = vnet_buffer (b[1])->ipsec.sad_index;
95 sai2 = vnet_buffer (b[2])->ipsec.sad_index;
96 sai3 = vnet_buffer (b[3])->ipsec.sad_index;
97 sa0 = pool_elt_at_index (im->sad, sai0);
98 sa1 = pool_elt_at_index (im->sad, sai1);
99 sa2 = pool_elt_at_index (im->sad, sai2);
100 sa3 = pool_elt_at_index (im->sad, sai3);
104 ti[0] = sa0->encrypt_thread_index;
105 ti[1] = sa1->encrypt_thread_index;
106 ti[2] = sa2->encrypt_thread_index;
107 ti[3] = sa3->encrypt_thread_index;
111 ti[0] = sa0->decrypt_thread_index;
112 ti[1] = sa1->decrypt_thread_index;
113 ti[2] = sa2->decrypt_thread_index;
114 ti[3] = sa3->decrypt_thread_index;
117 if (node->flags & VLIB_NODE_FLAG_TRACE)
119 if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
121 ipsec_handoff_trace_t *t =
122 vlib_add_trace (vm, node, b[0], sizeof (*t));
123 t->next_worker_index = ti[0];
125 if (PREDICT_FALSE (b[1]->flags & VLIB_BUFFER_IS_TRACED))
127 ipsec_handoff_trace_t *t =
128 vlib_add_trace (vm, node, b[1], sizeof (*t));
129 t->next_worker_index = ti[1];
131 if (PREDICT_FALSE (b[2]->flags & VLIB_BUFFER_IS_TRACED))
133 ipsec_handoff_trace_t *t =
134 vlib_add_trace (vm, node, b[2], sizeof (*t));
135 t->next_worker_index = ti[2];
137 if (PREDICT_FALSE (b[3]->flags & VLIB_BUFFER_IS_TRACED))
139 ipsec_handoff_trace_t *t =
140 vlib_add_trace (vm, node, b[3], sizeof (*t));
141 t->next_worker_index = ti[3];
149 while (n_left_from > 0)
154 sai0 = vnet_buffer (b[0])->ipsec.sad_index;
155 sa0 = pool_elt_at_index (im->sad, sai0);
158 ti[0] = sa0->encrypt_thread_index;
160 ti[0] = sa0->decrypt_thread_index;
162 if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
164 ipsec_handoff_trace_t *t =
165 vlib_add_trace (vm, node, b[0], sizeof (*t));
166 t->next_worker_index = ti[0];
174 n_enq = vlib_buffer_enqueue_to_thread (vm, fq_index, from,
175 thread_indices, frame->n_vectors, 1);
177 if (n_enq < frame->n_vectors)
178 vlib_node_increment_counter (vm, node->node_index,
179 IPSEC_HANDOFF_ERROR_CONGESTION_DROP,
180 frame->n_vectors - n_enq);
185 VLIB_NODE_FN (esp4_encrypt_handoff) (vlib_main_t * vm,
186 vlib_node_runtime_t * node,
187 vlib_frame_t * from_frame)
189 ipsec_main_t *im = &ipsec_main;
191 return ipsec_handoff (vm, node, from_frame, im->esp4_enc_fq_index, true);
194 VLIB_NODE_FN (esp6_encrypt_handoff) (vlib_main_t * vm,
195 vlib_node_runtime_t * node,
196 vlib_frame_t * from_frame)
198 ipsec_main_t *im = &ipsec_main;
200 return ipsec_handoff (vm, node, from_frame, im->esp6_enc_fq_index, true);
203 VLIB_NODE_FN (esp4_encrypt_tun_handoff) (vlib_main_t * vm,
204 vlib_node_runtime_t * node,
205 vlib_frame_t * from_frame)
207 ipsec_main_t *im = &ipsec_main;
209 return ipsec_handoff (vm, node, from_frame, im->esp4_enc_tun_fq_index,
213 VLIB_NODE_FN (esp6_encrypt_tun_handoff) (vlib_main_t * vm,
214 vlib_node_runtime_t * node,
215 vlib_frame_t * from_frame)
217 ipsec_main_t *im = &ipsec_main;
219 return ipsec_handoff (vm, node, from_frame, im->esp6_enc_tun_fq_index,
223 VLIB_NODE_FN (esp4_decrypt_handoff) (vlib_main_t * vm,
224 vlib_node_runtime_t * node,
225 vlib_frame_t * from_frame)
227 ipsec_main_t *im = &ipsec_main;
229 return ipsec_handoff (vm, node, from_frame, im->esp4_dec_fq_index, false);
232 VLIB_NODE_FN (esp6_decrypt_handoff) (vlib_main_t * vm,
233 vlib_node_runtime_t * node,
234 vlib_frame_t * from_frame)
236 ipsec_main_t *im = &ipsec_main;
238 return ipsec_handoff (vm, node, from_frame, im->esp6_dec_fq_index, false);
241 VLIB_NODE_FN (esp4_decrypt_tun_handoff) (vlib_main_t * vm,
242 vlib_node_runtime_t * node,
243 vlib_frame_t * from_frame)
245 ipsec_main_t *im = &ipsec_main;
247 return ipsec_handoff (vm, node, from_frame, im->esp4_dec_tun_fq_index,
251 VLIB_NODE_FN (esp6_decrypt_tun_handoff) (vlib_main_t * vm,
252 vlib_node_runtime_t * node,
253 vlib_frame_t * from_frame)
255 ipsec_main_t *im = &ipsec_main;
257 return ipsec_handoff (vm, node, from_frame, im->esp6_dec_tun_fq_index,
261 VLIB_NODE_FN (ah4_encrypt_handoff) (vlib_main_t * vm,
262 vlib_node_runtime_t * node,
263 vlib_frame_t * from_frame)
265 ipsec_main_t *im = &ipsec_main;
267 return ipsec_handoff (vm, node, from_frame, im->ah4_enc_fq_index, true);
270 VLIB_NODE_FN (ah6_encrypt_handoff) (vlib_main_t * vm,
271 vlib_node_runtime_t * node,
272 vlib_frame_t * from_frame)
274 ipsec_main_t *im = &ipsec_main;
276 return ipsec_handoff (vm, node, from_frame, im->ah6_enc_fq_index, true);
279 VLIB_NODE_FN (ah4_decrypt_handoff) (vlib_main_t * vm,
280 vlib_node_runtime_t * node,
281 vlib_frame_t * from_frame)
283 ipsec_main_t *im = &ipsec_main;
285 return ipsec_handoff (vm, node, from_frame, im->ah4_dec_fq_index, false);
288 VLIB_NODE_FN (ah6_decrypt_handoff) (vlib_main_t * vm,
289 vlib_node_runtime_t * node,
290 vlib_frame_t * from_frame)
292 ipsec_main_t *im = &ipsec_main;
294 return ipsec_handoff (vm, node, from_frame, im->ah6_dec_fq_index, false);
298 VLIB_REGISTER_NODE (esp4_encrypt_handoff) = {
299 .name = "esp4-encrypt-handoff",
300 .vector_size = sizeof (u32),
301 .format_trace = format_ipsec_handoff_trace,
302 .type = VLIB_NODE_TYPE_INTERNAL,
303 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
304 .error_strings = ipsec_handoff_error_strings,
310 VLIB_REGISTER_NODE (esp6_encrypt_handoff) = {
311 .name = "esp6-encrypt-handoff",
312 .vector_size = sizeof (u32),
313 .format_trace = format_ipsec_handoff_trace,
314 .type = VLIB_NODE_TYPE_INTERNAL,
315 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
316 .error_strings = ipsec_handoff_error_strings,
322 VLIB_REGISTER_NODE (esp4_encrypt_tun_handoff) = {
323 .name = "esp4-encrypt-tun-handoff",
324 .vector_size = sizeof (u32),
325 .format_trace = format_ipsec_handoff_trace,
326 .type = VLIB_NODE_TYPE_INTERNAL,
327 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
328 .error_strings = ipsec_handoff_error_strings,
334 VLIB_REGISTER_NODE (esp6_encrypt_tun_handoff) = {
335 .name = "esp6-encrypt-tun-handoff",
336 .vector_size = sizeof (u32),
337 .format_trace = format_ipsec_handoff_trace,
338 .type = VLIB_NODE_TYPE_INTERNAL,
339 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
340 .error_strings = ipsec_handoff_error_strings,
346 VLIB_REGISTER_NODE (esp4_decrypt_handoff) = {
347 .name = "esp4-decrypt-handoff",
348 .vector_size = sizeof (u32),
349 .format_trace = format_ipsec_handoff_trace,
350 .type = VLIB_NODE_TYPE_INTERNAL,
351 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
352 .error_strings = ipsec_handoff_error_strings,
358 VLIB_REGISTER_NODE (esp6_decrypt_handoff) = {
359 .name = "esp6-decrypt-handoff",
360 .vector_size = sizeof (u32),
361 .format_trace = format_ipsec_handoff_trace,
362 .type = VLIB_NODE_TYPE_INTERNAL,
363 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
364 .error_strings = ipsec_handoff_error_strings,
370 VLIB_REGISTER_NODE (esp4_decrypt_tun_handoff) = {
371 .name = "esp4-decrypt-tun-handoff",
372 .vector_size = sizeof (u32),
373 .format_trace = format_ipsec_handoff_trace,
374 .type = VLIB_NODE_TYPE_INTERNAL,
375 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
376 .error_strings = ipsec_handoff_error_strings,
382 VLIB_REGISTER_NODE (esp6_decrypt_tun_handoff) = {
383 .name = "esp6-decrypt-tun-handoff",
384 .vector_size = sizeof (u32),
385 .format_trace = format_ipsec_handoff_trace,
386 .type = VLIB_NODE_TYPE_INTERNAL,
387 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
388 .error_strings = ipsec_handoff_error_strings,
394 VLIB_REGISTER_NODE (ah4_encrypt_handoff) = {
395 .name = "ah4-encrypt-handoff",
396 .vector_size = sizeof (u32),
397 .format_trace = format_ipsec_handoff_trace,
398 .type = VLIB_NODE_TYPE_INTERNAL,
399 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
400 .error_strings = ipsec_handoff_error_strings,
406 VLIB_REGISTER_NODE (ah6_encrypt_handoff) = {
407 .name = "ah6-encrypt-handoff",
408 .vector_size = sizeof (u32),
409 .format_trace = format_ipsec_handoff_trace,
410 .type = VLIB_NODE_TYPE_INTERNAL,
411 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
412 .error_strings = ipsec_handoff_error_strings,
418 VLIB_REGISTER_NODE (ah4_decrypt_handoff) = {
419 .name = "ah4-decrypt-handoff",
420 .vector_size = sizeof (u32),
421 .format_trace = format_ipsec_handoff_trace,
422 .type = VLIB_NODE_TYPE_INTERNAL,
423 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
424 .error_strings = ipsec_handoff_error_strings,
430 VLIB_REGISTER_NODE (ah6_decrypt_handoff) = {
431 .name = "ah6-decrypt-handoff",
432 .vector_size = sizeof (u32),
433 .format_trace = format_ipsec_handoff_trace,
434 .type = VLIB_NODE_TYPE_INTERNAL,
435 .n_errors = ARRAY_LEN(ipsec_handoff_error_strings),
436 .error_strings = ipsec_handoff_error_strings,
445 * fd.io coding-style-patch-verification: ON
448 * eval: (c-set-style "gnu")