2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
18 #include <vnet/ip/ip.h>
19 #include <vnet/crypto/crypto.h>
20 #include <vnet/feature/feature.h>
22 #include <vppinfra/types.h>
23 #include <vppinfra/cache.h>
25 #include <vnet/ipsec/ipsec_spd.h>
26 #include <vnet/ipsec/ipsec_spd_policy.h>
27 #include <vnet/ipsec/ipsec_sa.h>
29 #include <vppinfra/bihash_8_16.h>
31 #include <vppinfra/bihash_24_16.h>
33 #define IPSEC_FP_IP4_HASHES_POOL_SIZE 128
34 #define IPSEC_FP_IP6_HASHES_POOL_SIZE 128
36 typedef clib_error_t *(*add_del_sa_sess_cb_t) (u32 sa_index, u8 is_add);
37 typedef clib_error_t *(*check_support_cb_t) (ipsec_sa_t * sa);
38 typedef clib_error_t *(*enable_disable_cb_t) (int is_enable);
42 u64 key[2]; // 16 bytes
46 } ipsec4_hash_kv_16_8_t;
52 ip4_address_t ip4_addr[2];
57 ipsec4_hash_kv_16_8_t kv_16_8;
58 } ipsec4_spd_5tuple_t;
64 ip4_address_t ip4_src_addr;
65 ip4_address_t ip4_dest_addr;
66 ipsec_spd_policy_type_t policy_type;
69 ipsec4_hash_kv_16_8_t kv_16_8;
70 } ipsec4_inbound_spd_tuple_t;
75 /* add/del callback */
76 add_del_sa_sess_cb_t add_del_sa_sess_cb;
77 /* check support function */
78 check_support_cb_t check_support_cb;
79 u32 ah4_encrypt_node_index;
80 u32 ah4_decrypt_node_index;
81 u32 ah4_encrypt_next_index;
82 u32 ah4_decrypt_next_index;
83 u32 ah6_encrypt_node_index;
84 u32 ah6_decrypt_node_index;
85 u32 ah6_encrypt_next_index;
86 u32 ah6_decrypt_next_index;
92 /* add/del callback */
93 add_del_sa_sess_cb_t add_del_sa_sess_cb;
94 /* check support function */
95 check_support_cb_t check_support_cb;
96 u32 esp4_encrypt_node_index;
97 u32 esp4_decrypt_node_index;
98 u32 esp4_encrypt_next_index;
99 u32 esp4_decrypt_next_index;
100 u32 esp6_encrypt_node_index;
101 u32 esp6_decrypt_node_index;
102 u32 esp6_encrypt_next_index;
103 u32 esp6_decrypt_next_index;
104 u32 esp4_decrypt_tun_node_index;
105 u32 esp4_decrypt_tun_next_index;
106 u32 esp4_encrypt_tun_node_index;
107 u32 esp6_decrypt_tun_node_index;
108 u32 esp6_decrypt_tun_next_index;
109 u32 esp6_encrypt_tun_node_index;
110 u32 esp_mpls_encrypt_tun_node_index;
111 } ipsec_esp_backend_t;
115 vnet_crypto_op_id_t enc_op_id;
116 vnet_crypto_op_id_t dec_op_id;
117 vnet_crypto_alg_t alg;
121 } ipsec_main_crypto_alg_t;
125 vnet_crypto_op_id_t op_id;
126 vnet_crypto_alg_t alg;
128 } ipsec_main_integ_alg_t;
132 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
133 vnet_crypto_op_t *crypto_ops;
134 vnet_crypto_op_t *integ_ops;
135 vnet_crypto_op_t *chained_crypto_ops;
136 vnet_crypto_op_t *chained_integ_ops;
137 vnet_crypto_op_chunk_t *chunks;
138 vnet_crypto_async_frame_t **async_frames;
139 } ipsec_per_thread_data_t;
143 /* pool of tunnel instances */
145 /* pool of policies */
146 ipsec_policy_t *policies;
147 /* pool of bihash tables for ipv4 ipsec rules */
148 clib_bihash_16_8_t *fp_ip4_lookup_hashes_pool;
149 /* pool of bihash tables for ipv6 ipsec rules */
150 clib_bihash_40_8_t *fp_ip6_lookup_hashes_pool;
152 u32 fp_spd_ipv4_out_is_enabled;
153 u32 fp_spd_ipv4_in_is_enabled;
154 u32 fp_spd_ipv6_out_is_enabled;
155 u32 fp_spd_ipv6_in_is_enabled;
156 /* pool of fast path mask types */
157 ipsec_fp_mask_type_entry_t *fp_mask_types;
158 u32 fp_lookup_hash_buckets; /* number of buckets should be power of two */
160 /* hash tables of UDP port registrations */
161 uword *udp_port_registrations;
163 uword *tunnel_index_by_key;
165 /* next_header protocol registration */
166 u16 *next_header_registrations;
169 vlib_main_t *vlib_main;
170 vnet_main_t *vnet_main;
173 uword *spd_index_by_spd_id;
174 uword *spd_index_by_sw_if_index;
175 uword *sa_index_by_sa_id;
176 uword *ipsec4_if_pool_index_by_key;
177 uword *ipsec6_if_pool_index_by_key;
178 uword *ipsec_if_real_dev_by_show_dev;
179 uword *ipsec_if_by_sw_if_index;
181 ipsec4_hash_kv_16_8_t *ipsec4_out_spd_hash_tbl;
182 ipsec4_hash_kv_16_8_t *ipsec4_in_spd_hash_tbl;
183 clib_bihash_8_16_t tun4_protect_by_key;
184 clib_bihash_24_16_t tun6_protect_by_key;
187 u32 error_drop_node_index;
188 u32 esp4_encrypt_node_index;
189 u32 esp4_decrypt_node_index;
190 u32 esp4_decrypt_tun_node_index;
191 u32 esp4_encrypt_tun_node_index;
192 u32 ah4_encrypt_node_index;
193 u32 ah4_decrypt_node_index;
194 u32 esp6_encrypt_node_index;
195 u32 esp6_decrypt_node_index;
196 u32 esp6_decrypt_tun_node_index;
197 u32 esp6_encrypt_tun_node_index;
198 u32 esp_mpls_encrypt_tun_node_index;
199 u32 ah6_encrypt_node_index;
200 u32 ah6_decrypt_node_index;
201 /* next node indices */
202 u32 esp4_encrypt_next_index;
203 u32 esp4_decrypt_next_index;
204 u32 esp4_decrypt_tun_next_index;
205 u32 ah4_encrypt_next_index;
206 u32 ah4_decrypt_next_index;
207 u32 esp6_encrypt_next_index;
208 u32 esp6_decrypt_next_index;
209 u32 esp6_decrypt_tun_next_index;
210 u32 ah6_encrypt_next_index;
211 u32 ah6_decrypt_next_index;
213 /* pool of ah backends */
214 ipsec_ah_backend_t *ah_backends;
215 /* pool of esp backends */
216 ipsec_esp_backend_t *esp_backends;
217 /* index of current ah backend */
218 u32 ah_current_backend;
219 /* index of current esp backend */
220 u32 esp_current_backend;
221 /* index of default ah backend */
222 u32 ah_default_backend;
223 /* index of default esp backend */
224 u32 esp_default_backend;
226 /* crypto alg data */
227 ipsec_main_crypto_alg_t *crypto_algs;
229 /* crypto integ data */
230 ipsec_main_integ_alg_t *integ_algs;
232 /* per-thread data */
233 ipsec_per_thread_data_t *ptd;
235 /** Worker handoff */
236 u32 ah4_enc_fq_index;
237 u32 ah4_dec_fq_index;
238 u32 ah6_enc_fq_index;
239 u32 ah6_dec_fq_index;
241 u32 esp4_enc_fq_index;
242 u32 esp4_dec_fq_index;
243 u32 esp6_enc_fq_index;
244 u32 esp6_dec_fq_index;
245 u32 esp4_enc_tun_fq_index;
246 u32 esp6_enc_tun_fq_index;
247 u32 esp_mpls_enc_tun_fq_index;
248 u32 esp4_dec_tun_fq_index;
249 u32 esp6_dec_tun_fq_index;
251 /* Number of buckets for flow cache */
252 u32 ipsec4_out_spd_hash_num_buckets;
253 u32 ipsec4_out_spd_flow_cache_entries;
255 u8 output_flow_cache_flag;
257 u32 ipsec4_in_spd_hash_num_buckets;
258 u32 ipsec4_in_spd_flow_cache_entries;
259 u32 input_epoch_count;
260 u8 input_flow_cache_flag;
266 typedef enum ipsec_format_flags_t_
268 IPSEC_FORMAT_BRIEF = 0,
269 IPSEC_FORMAT_DETAIL = (1 << 0),
270 IPSEC_FORMAT_INSECURE = (1 << 1),
271 } ipsec_format_flags_t;
273 extern ipsec_main_t ipsec_main;
275 clib_error_t *ipsec_add_del_sa_sess_cb (ipsec_main_t * im, u32 sa_index,
278 clib_error_t *ipsec_check_support_cb (ipsec_main_t * im, ipsec_sa_t * sa);
280 extern vlib_node_registration_t ipsec4_tun_input_node;
281 extern vlib_node_registration_t ipsec6_tun_input_node;
291 static_always_inline u32
292 get_next_output_feature_node_index (vlib_buffer_t * b,
293 vlib_node_runtime_t * nr)
296 vlib_main_t *vm = vlib_get_main ();
297 vlib_node_t *node = vlib_get_node (vm, nr->node_index);
299 vnet_feature_next (&next, b);
300 return node->next_nodes[next];
303 static_always_inline u64
304 ipsec4_hash_16_8 (ipsec4_hash_kv_16_8_t *v)
306 #ifdef clib_crc32c_uses_intrinsics
307 return clib_crc32c ((u8 *) v->key, 16);
309 u64 tmp = v->key[0] ^ v->key[1];
310 return clib_xxhash (tmp);
314 static_always_inline int
315 ipsec4_hash_key_compare_16_8 (u64 *a, u64 *b)
317 #if defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_UNALIGNED_LOAD_STORE)
319 v = u64x2_load_unaligned (a) ^ u64x2_load_unaligned (b);
320 return u64x2_is_all_zero (v);
322 return ((a[0] ^ b[0]) | (a[1] ^ b[1])) == 0;
326 /* clib_spinlock_lock is not used to save another memory indirection */
327 static_always_inline void
328 ipsec_spinlock_lock (i32 *lock)
331 while (!clib_atomic_cmp_and_swap_acq_relax_n (lock, &free, 1, 0))
333 /* atomic load limits number of compare_exchange executions */
334 while (clib_atomic_load_relax_n (lock))
336 /* on failure, compare_exchange writes lock into free */
341 static_always_inline void
342 ipsec_spinlock_unlock (i32 *lock)
344 /* Make sure all reads/writes are complete before releasing the lock */
345 clib_atomic_release (lock);
348 /* Special case to drop or hand off packets for sync/async modes.
350 * Different than sync mode, async mode only enqueue drop or hand-off packets
354 ipsec_set_next_index (vlib_buffer_t *b, vlib_node_runtime_t *node,
355 u32 thread_index, u32 err, u32 ipsec_sa_err, u16 index,
356 u16 *nexts, u16 drop_next, u32 sa_index)
358 nexts[index] = drop_next;
359 b->error = node->errors[err];
360 if (PREDICT_TRUE (ipsec_sa_err != ~0))
361 vlib_increment_simple_counter (&ipsec_sa_err_counters[ipsec_sa_err],
362 thread_index, sa_index, 1);
365 u32 ipsec_register_ah_backend (vlib_main_t * vm, ipsec_main_t * im,
367 const char *ah4_encrypt_node_name,
368 const char *ah4_decrypt_node_name,
369 const char *ah6_encrypt_node_name,
370 const char *ah6_decrypt_node_name,
371 check_support_cb_t ah_check_support_cb,
372 add_del_sa_sess_cb_t ah_add_del_sa_sess_cb);
374 u32 ipsec_register_esp_backend (
375 vlib_main_t *vm, ipsec_main_t *im, const char *name,
376 const char *esp4_encrypt_node_name, const char *esp4_encrypt_tun_node_name,
377 const char *esp4_decrypt_node_name, const char *esp4_decrypt_tun_node_name,
378 const char *esp6_encrypt_node_name, const char *esp6_encrypt_tun_node_name,
379 const char *esp6_decrypt_node_name, const char *esp6_decrypt_tun_node_name,
380 const char *esp_mpls_encrypt_tun_node_name,
381 check_support_cb_t esp_check_support_cb,
382 add_del_sa_sess_cb_t esp_add_del_sa_sess_cb);
384 int ipsec_select_ah_backend (ipsec_main_t * im, u32 ah_backend_idx);
385 int ipsec_select_esp_backend (ipsec_main_t * im, u32 esp_backend_idx);
387 clib_error_t *ipsec_rsc_in_use (ipsec_main_t * im);
388 void ipsec_set_async_mode (u32 is_enabled);
390 extern void ipsec_register_udp_port (u16 udp_port, u8 is_ip4);
391 extern void ipsec_unregister_udp_port (u16 udp_port, u8 is_ip4);
393 extern clib_error_t *ipsec_register_next_header (vlib_main_t *vm,
395 const char *next_node);
397 #endif /* __IPSEC_H__ */
400 * fd.io coding-style-patch-verification: ON
403 * eval: (c-set-style "gnu")