2 * ipsec_output.c : IPSec output node
4 * Copyright (c) 2015 Cisco and/or its affiliates.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at:
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
18 #include <vnet/vnet.h>
19 #include <vnet/api_errno.h>
20 #include <vnet/ip/ip.h>
22 #include <vnet/ipsec/ipsec.h>
23 #include <vnet/ipsec/ipsec_io.h>
25 #define foreach_ipsec_output_error \
26 _(RX_PKTS, "IPSec pkts received") \
27 _(POLICY_DISCARD, "IPSec policy discard") \
28 _(POLICY_NO_MATCH, "IPSec policy (no match)") \
29 _(POLICY_PROTECT, "IPSec policy protect") \
30 _(POLICY_BYPASS, "IPSec policy bypass") \
31 _(ENCAPS_FAILED, "IPSec encapsulation failed")
35 #define _(sym,str) IPSEC_OUTPUT_ERROR_##sym,
36 foreach_ipsec_output_error
39 } ipsec_output_error_t;
41 static char *ipsec_output_error_strings[] = {
42 #define _(sym,string) string,
43 foreach_ipsec_output_error
51 } ipsec_output_trace_t;
53 /* packet trace format function */
55 format_ipsec_output_trace (u8 * s, va_list * args)
57 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
58 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
59 ipsec_output_trace_t *t = va_arg (*args, ipsec_output_trace_t *);
61 s = format (s, "spd %u policy %d", t->spd_id, t->policy_id);
67 ipsec4_out_spd_add_flow_cache_entry (ipsec_main_t *im, u8 pr, u32 la, u32 ra,
68 u16 lp, u16 rp, u32 pol_id)
71 u8 overwrite = 0, stale_overwrite = 0;
72 ipsec4_spd_5tuple_t ip4_5tuple = { .ip4_addr = { (ip4_address_t) la,
77 ip4_5tuple.kv_16_8.value = (((u64) pol_id) << 32) | ((u64) im->epoch_count);
79 hash = ipsec4_hash_16_8 (&ip4_5tuple.kv_16_8);
80 hash &= (im->ipsec4_out_spd_hash_num_buckets - 1);
82 ipsec_spinlock_lock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
83 /* Check if we are overwriting an existing entry so we know
84 whether to increment the flow cache counter. Since flow
85 cache counter is reset on any policy add/remove, but
86 hash table values are not, we also need to check if the entry
87 we are overwriting is stale or not. If it's a stale entry
88 overwrite, we still want to increment flow cache counter */
89 overwrite = (im->ipsec4_out_spd_hash_tbl[hash].value != 0);
90 /* Check for stale entry by comparing with current epoch count */
91 if (PREDICT_FALSE (overwrite))
94 ((u32) (im->ipsec4_out_spd_hash_tbl[hash].value & 0xFFFFFFFF)));
95 clib_memcpy_fast (&im->ipsec4_out_spd_hash_tbl[hash], &ip4_5tuple.kv_16_8,
96 sizeof (ip4_5tuple.kv_16_8));
97 ipsec_spinlock_unlock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
99 /* Increment the counter to track active flow cache entries
100 when entering a fresh entry or overwriting a stale one */
101 if (!overwrite || stale_overwrite)
102 clib_atomic_fetch_add_relax (&im->ipsec4_out_spd_flow_cache_entries, 1);
107 always_inline ipsec_policy_t *
108 ipsec4_out_spd_find_flow_cache_entry (ipsec_main_t *im, u8 pr, u32 la, u32 ra,
111 ipsec_policy_t *p = NULL;
112 ipsec4_hash_kv_16_8_t kv_result;
115 if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
116 (pr != IP_PROTOCOL_SCTP)))
121 ipsec4_spd_5tuple_t ip4_5tuple = { .ip4_addr = { (ip4_address_t) la,
122 (ip4_address_t) ra },
126 hash = ipsec4_hash_16_8 (&ip4_5tuple.kv_16_8);
127 hash &= (im->ipsec4_out_spd_hash_num_buckets - 1);
129 ipsec_spinlock_lock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
130 kv_result = im->ipsec4_out_spd_hash_tbl[hash];
131 ipsec_spinlock_unlock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
133 if (ipsec4_hash_key_compare_16_8 ((u64 *) &ip4_5tuple.kv_16_8,
136 if (im->epoch_count == ((u32) (kv_result.value & 0xFFFFFFFF)))
138 /* Get the policy based on the index */
140 pool_elt_at_index (im->policies, ((u32) (kv_result.value >> 32)));
147 always_inline ipsec_policy_t *
148 ipsec_output_policy_match (ipsec_spd_t *spd, u8 pr, u32 la, u32 ra, u16 lp,
149 u16 rp, u8 flow_cache_enabled)
151 ipsec_main_t *im = &ipsec_main;
158 vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP4_OUTBOUND])
160 p = pool_elt_at_index (im->policies, *i);
161 if (PREDICT_FALSE (p->protocol && (p->protocol != pr)))
164 if (ra < clib_net_to_host_u32 (p->raddr.start.ip4.as_u32))
167 if (ra > clib_net_to_host_u32 (p->raddr.stop.ip4.as_u32))
170 if (la < clib_net_to_host_u32 (p->laddr.start.ip4.as_u32))
173 if (la > clib_net_to_host_u32 (p->laddr.stop.ip4.as_u32))
176 if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
177 (pr != IP_PROTOCOL_SCTP)))
184 if (lp < p->lport.start)
187 if (lp > p->lport.stop)
190 if (rp < p->rport.start)
193 if (rp > p->rport.stop)
197 if (flow_cache_enabled)
199 /* Add an Entry in Flow cache */
200 ipsec4_out_spd_add_flow_cache_entry (
201 im, pr, clib_host_to_net_u32 (la), clib_host_to_net_u32 (ra),
202 clib_host_to_net_u16 (lp), clib_host_to_net_u16 (rp), *i);
211 ip6_addr_match_range (ip6_address_t * a, ip6_address_t * la,
214 if ((memcmp (a->as_u64, la->as_u64, 2 * sizeof (u64)) >= 0) &&
215 (memcmp (a->as_u64, ua->as_u64, 2 * sizeof (u64)) <= 0))
220 always_inline ipsec_policy_t *
221 ipsec6_output_policy_match (ipsec_spd_t * spd,
223 ip6_address_t * ra, u16 lp, u16 rp, u8 pr)
225 ipsec_main_t *im = &ipsec_main;
232 vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP6_OUTBOUND])
234 p = pool_elt_at_index (im->policies, *i);
235 if (PREDICT_FALSE (p->protocol && (p->protocol != pr)))
238 if (!ip6_addr_match_range (ra, &p->raddr.start.ip6, &p->raddr.stop.ip6))
241 if (!ip6_addr_match_range (la, &p->laddr.start.ip6, &p->laddr.stop.ip6))
245 ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP)
246 && (pr != IP_PROTOCOL_SCTP)))
249 if (lp < p->lport.start)
252 if (lp > p->lport.stop)
255 if (rp < p->rport.start)
258 if (rp > p->rport.stop)
268 ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
269 vlib_frame_t * from_frame, int is_ipv6)
271 ipsec_main_t *im = &ipsec_main;
273 u32 *from, *to_next = 0, thread_index;
274 u32 n_left_from, sw_if_index0, last_sw_if_index = (u32) ~ 0;
275 u32 next_node_index = (u32) ~ 0, last_next_node_index = (u32) ~ 0;
278 ipsec_spd_t *spd0 = 0;
280 u64 nc_protect = 0, nc_bypass = 0, nc_discard = 0, nc_nomatch = 0;
281 u8 flow_cache_enabled = im->flow_cache_flag;
283 from = vlib_frame_vector_args (from_frame);
284 n_left_from = from_frame->n_vectors;
285 thread_index = vm->thread_index;
287 while (n_left_from > 0)
290 vlib_buffer_t *b0, *b1;
291 ipsec_policy_t *p0 = NULL;
293 ip6_header_t *ip6_0 = 0;
300 b0 = vlib_get_buffer (vm, bi0);
304 b1 = vlib_get_buffer (vm, bi1);
305 CLIB_PREFETCH (b1, CLIB_CACHE_LINE_BYTES * 2, STORE);
306 vlib_prefetch_buffer_data (b1, LOAD);
308 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
309 iph_offset = vnet_buffer (b0)->ip.save_rewrite_length;
310 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0)
313 /* lookup for SPD only if sw_if_index is changed */
314 if (PREDICT_FALSE (last_sw_if_index != sw_if_index0))
316 uword *p = hash_get (im->spd_index_by_sw_if_index, sw_if_index0);
319 spd0 = pool_elt_at_index (im->spds, spd_index0);
320 last_sw_if_index = sw_if_index0;
325 ip6_0 = (ip6_header_t *) ((u8 *) vlib_buffer_get_current (b0)
328 udp0 = ip6_next_header (ip6_0);
331 ("packet received from %U port %u to %U port %u spd_id %u",
332 format_ip6_address, &ip6_0->src_address,
333 clib_net_to_host_u16 (udp0->src_port), format_ip6_address,
334 &ip6_0->dst_address, clib_net_to_host_u16 (udp0->dst_port),
338 p0 = ipsec6_output_policy_match (spd0,
344 (udp0->dst_port), ip6_0->protocol);
348 udp0 = (udp_header_t *) ((u8 *) ip0 + ip4_header_bytes (ip0));
351 clib_warning ("packet received from %U to %U port %u",
352 format_ip4_address, ip0->src_address.as_u8,
353 format_ip4_address, ip0->dst_address.as_u8,
354 clib_net_to_host_u16 (udp0->dst_port));
355 clib_warning ("sw_if_index0 %u spd_index0 %u spd_id %u",
356 sw_if_index0, spd_index0, spd0->id);
360 * Check whether flow cache is enabled.
362 if (flow_cache_enabled)
364 p0 = ipsec4_out_spd_find_flow_cache_entry (
365 im, ip0->protocol, ip0->src_address.as_u32,
366 ip0->dst_address.as_u32, udp0->src_port, udp0->dst_port);
369 /* Fall back to linear search if flow cache lookup fails */
372 p0 = ipsec_output_policy_match (
374 clib_net_to_host_u32 (ip0->src_address.as_u32),
375 clib_net_to_host_u32 (ip0->dst_address.as_u32),
376 clib_net_to_host_u16 (udp0->src_port),
377 clib_net_to_host_u16 (udp0->dst_port), flow_cache_enabled);
380 tcp0 = (void *) udp0;
382 if (PREDICT_TRUE (p0 != NULL))
384 pi0 = p0 - im->policies;
386 vlib_prefetch_combined_counter (&ipsec_spd_policy_counters,
391 bytes0 = clib_net_to_host_u16 (ip6_0->payload_length);
392 bytes0 += sizeof (ip6_header_t);
396 bytes0 = clib_net_to_host_u16 (ip0->length);
399 if (p0->policy == IPSEC_POLICY_ACTION_PROTECT)
403 sa = ipsec_sa_get (p0->sa_index);
404 if (sa->protocol == IPSEC_PROTOCOL_ESP)
406 next_node_index = im->esp6_encrypt_node_index;
408 next_node_index = im->esp4_encrypt_node_index;
410 next_node_index = im->ah6_encrypt_node_index;
412 next_node_index = im->ah4_encrypt_node_index;
413 vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
415 if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD))
417 vnet_buffer_oflags_t oflags = vnet_buffer (b0)->oflags;
420 * Clearing offload flags before checksum is computed
421 * It guarantees the cache hit!
423 vnet_buffer_offload_flags_clear (b0, oflags);
427 if (PREDICT_FALSE (oflags &
428 VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
430 tcp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
431 vm, b0, ip6_0, &bogus);
433 if (PREDICT_FALSE (oflags &
434 VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
436 udp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
437 vm, b0, ip6_0, &bogus);
442 if (PREDICT_FALSE (oflags &
443 VNET_BUFFER_OFFLOAD_F_IP_CKSUM))
445 ip0->checksum = ip4_header_checksum (ip0);
447 if (PREDICT_FALSE (oflags &
448 VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
451 ip4_tcp_udp_compute_checksum (vm, b0, ip0);
453 if (PREDICT_FALSE (oflags &
454 VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
457 ip4_tcp_udp_compute_checksum (vm, b0, ip0);
461 vlib_buffer_advance (b0, iph_offset);
463 else if (p0->policy == IPSEC_POLICY_ACTION_BYPASS)
466 next_node_index = get_next_output_feature_node_index (b0, node);
471 next_node_index = im->error_drop_node_index;
473 vlib_increment_combined_counter
474 (&ipsec_spd_policy_counters, thread_index, pi0, 1, bytes0);
480 next_node_index = im->error_drop_node_index;
486 if (PREDICT_FALSE ((last_next_node_index != next_node_index) || f == 0))
488 /* if this is not 1st frame */
490 vlib_put_frame_to_node (vm, last_next_node_index, f);
492 last_next_node_index = next_node_index;
494 f = vlib_get_frame_to_node (vm, next_node_index);
496 /* frame->frame_flags, copy it from node */
497 /* Copy trace flag from next_frame and from runtime. */
498 f->frame_flags |= node->flags & VLIB_NODE_FLAG_TRACE;
500 to_next = vlib_frame_vector_args (f);
507 if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
508 PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
510 ipsec_output_trace_t *tr =
511 vlib_add_trace (vm, node, b0, sizeof (*tr));
513 tr->spd_id = spd0->id;
518 vlib_put_frame_to_node (vm, next_node_index, f);
519 vlib_node_increment_counter (vm, node->node_index,
520 IPSEC_OUTPUT_ERROR_POLICY_PROTECT, nc_protect);
521 vlib_node_increment_counter (vm, node->node_index,
522 IPSEC_OUTPUT_ERROR_POLICY_BYPASS, nc_bypass);
523 vlib_node_increment_counter (vm, node->node_index,
524 IPSEC_OUTPUT_ERROR_POLICY_DISCARD, nc_discard);
525 vlib_node_increment_counter (vm, node->node_index,
526 IPSEC_OUTPUT_ERROR_POLICY_NO_MATCH,
528 return from_frame->n_vectors;
531 VLIB_NODE_FN (ipsec4_output_node) (vlib_main_t * vm,
532 vlib_node_runtime_t * node,
533 vlib_frame_t * frame)
535 return ipsec_output_inline (vm, node, frame, 0);
539 VLIB_REGISTER_NODE (ipsec4_output_node) = {
540 .name = "ipsec4-output-feature",
541 .vector_size = sizeof (u32),
542 .format_trace = format_ipsec_output_trace,
543 .type = VLIB_NODE_TYPE_INTERNAL,
545 .n_errors = ARRAY_LEN(ipsec_output_error_strings),
546 .error_strings = ipsec_output_error_strings,
548 .n_next_nodes = IPSEC_OUTPUT_N_NEXT,
550 #define _(s,n) [IPSEC_OUTPUT_NEXT_##s] = n,
551 foreach_ipsec_output_next
557 VLIB_NODE_FN (ipsec6_output_node) (vlib_main_t * vm,
558 vlib_node_runtime_t * node,
559 vlib_frame_t * frame)
561 return ipsec_output_inline (vm, node, frame, 1);
564 VLIB_REGISTER_NODE (ipsec6_output_node) = {
565 .name = "ipsec6-output-feature",
566 .vector_size = sizeof (u32),
567 .format_trace = format_ipsec_output_trace,
568 .type = VLIB_NODE_TYPE_INTERNAL,
570 .n_errors = ARRAY_LEN(ipsec_output_error_strings),
571 .error_strings = ipsec_output_error_strings,
573 .n_next_nodes = IPSEC_OUTPUT_N_NEXT,
575 #define _(s,n) [IPSEC_OUTPUT_NEXT_##s] = n,
576 foreach_ipsec_output_next