2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/fib/ip4_fib.h>
24 #include <snat/snat.h>
25 #include <snat/snat_ipfix_logging.h>
26 #include <snat/snat_det.h>
28 #include <vppinfra/hash.h>
29 #include <vppinfra/error.h>
30 #include <vppinfra/elog.h>
37 } snat_in2out_trace_t;
40 u32 next_worker_index;
42 } snat_in2out_worker_handoff_trace_t;
44 /* packet trace format function */
45 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
47 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
48 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
49 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
52 tag = t->is_slow_path ? "SNAT_IN2OUT_SLOW_PATH" : "SNAT_IN2OUT_FAST_PATH";
54 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
55 t->sw_if_index, t->next_index, t->session_index);
60 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
62 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
63 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
64 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
66 s = format (s, "SANT_IN2OUT_FAST: sw_if_index %d, next index %d",
67 t->sw_if_index, t->next_index);
72 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
74 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
75 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
76 snat_in2out_worker_handoff_trace_t * t =
77 va_arg (*args, snat_in2out_worker_handoff_trace_t *);
80 m = t->do_handoff ? "next worker" : "same worker";
81 s = format (s, "SNAT_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
86 vlib_node_registration_t snat_in2out_node;
87 vlib_node_registration_t snat_in2out_slowpath_node;
88 vlib_node_registration_t snat_in2out_fast_node;
89 vlib_node_registration_t snat_in2out_worker_handoff_node;
90 vlib_node_registration_t snat_det_in2out_node;
91 vlib_node_registration_t snat_in2out_output_node;
92 vlib_node_registration_t snat_in2out_output_slowpath_node;
93 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
94 vlib_node_registration_t snat_hairpin_dst_node;
95 vlib_node_registration_t snat_hairpin_src_node;
98 #define foreach_snat_in2out_error \
99 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
100 _(IN2OUT_PACKETS, "Good in2out packets processed") \
101 _(OUT_OF_PORTS, "Out of ports") \
102 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \
103 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
104 _(NO_TRANSLATION, "No translation")
107 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
108 foreach_snat_in2out_error
111 } snat_in2out_error_t;
113 static char * snat_in2out_error_strings[] = {
114 #define _(sym,string) string,
115 foreach_snat_in2out_error
120 SNAT_IN2OUT_NEXT_LOOKUP,
121 SNAT_IN2OUT_NEXT_DROP,
122 SNAT_IN2OUT_NEXT_ICMP_ERROR,
123 SNAT_IN2OUT_NEXT_SLOW_PATH,
125 } snat_in2out_next_t;
128 SNAT_HAIRPIN_SRC_NEXT_DROP,
129 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
130 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
131 SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
132 SNAT_HAIRPIN_SRC_N_NEXT,
133 } snat_hairpin_next_t;
136 * @brief Check if packet should be translated
138 * Packets aimed at outside interface and external addresss with active session
139 * should be translated.
141 * @param sm SNAT main
142 * @param rt SNAT runtime data
143 * @param sw_if_index0 index of the inside interface
144 * @param ip0 IPv4 header
145 * @param proto0 SNAT protocol
146 * @param rx_fib_index0 RX FIB index
148 * @returns 0 if packet should be translated otherwise 1
151 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
152 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
155 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
157 .fp_proto = FIB_PROTOCOL_IP4,
160 .ip4.as_u32 = ip0->dst_address.as_u32,
164 /* Don't NAT packet aimed at the intfc address */
165 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
166 ip0->dst_address.as_u32)))
169 fei = fib_table_lookup (rx_fib_index0, &pfx);
170 if (FIB_NODE_INDEX_INVALID != fei)
172 u32 sw_if_index = fib_entry_get_resolving_interface (fei);
173 if (sw_if_index == ~0)
175 fei = fib_table_lookup (sm->outside_fib_index, &pfx);
176 if (FIB_NODE_INDEX_INVALID != fei)
177 sw_if_index = fib_entry_get_resolving_interface (fei);
180 pool_foreach (i, sm->interfaces,
182 /* NAT packet aimed at outside interface */
183 if ((i->is_inside == 0) && (sw_if_index == i->sw_if_index))
192 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
193 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
196 udp_header_t * udp0 = ip4_next_header (ip0);
197 snat_session_key_t key0, sm0;
198 clib_bihash_kv_8_8_t kv0, value0;
200 key0.addr = ip0->dst_address;
201 key0.port = udp0->dst_port;
202 key0.protocol = proto0;
203 key0.fib_index = sm->outside_fib_index;
204 kv0.key = key0.as_u64;
206 /* NAT packet aimed at external address if */
207 /* has active sessions */
208 if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
210 /* or is static mappings */
211 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
217 return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
221 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
224 snat_session_key_t * key0,
225 snat_session_t ** sessionp,
226 vlib_node_runtime_t * node,
231 snat_user_key_t user_key;
233 clib_bihash_kv_8_8_t kv0, value0;
234 u32 oldest_per_user_translation_list_index;
235 dlist_elt_t * oldest_per_user_translation_list_elt;
236 dlist_elt_t * per_user_translation_list_elt;
237 dlist_elt_t * per_user_list_head_elt;
239 snat_session_key_t key1;
240 u32 address_index = ~0;
241 u32 outside_fib_index;
243 snat_worker_key_t worker_by_out_key;
245 p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
248 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
249 return SNAT_IN2OUT_NEXT_DROP;
251 outside_fib_index = p[0];
253 key1.protocol = key0->protocol;
254 user_key.addr = ip0->src_address;
255 user_key.fib_index = rx_fib_index0;
256 kv0.key = user_key.as_u64;
258 /* Ever heard of the "user" = src ip4 address before? */
259 if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
261 /* no, make a new one */
262 pool_get (sm->per_thread_data[thread_index].users, u);
263 memset (u, 0, sizeof (*u));
264 u->addr = ip0->src_address;
265 u->fib_index = rx_fib_index0;
267 pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt);
269 u->sessions_per_user_list_head_index = per_user_list_head_elt -
270 sm->per_thread_data[thread_index].list_pool;
272 clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
273 u->sessions_per_user_list_head_index);
275 kv0.value = u - sm->per_thread_data[thread_index].users;
278 clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
282 u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
286 /* Over quota? Recycle the least recently used dynamic translation */
287 if (u->nsessions >= sm->max_translations_per_user)
289 /* Remove the oldest dynamic translation */
291 oldest_per_user_translation_list_index =
292 clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool,
293 u->sessions_per_user_list_head_index);
295 ASSERT (oldest_per_user_translation_list_index != ~0);
297 /* add it back to the end of the LRU list */
298 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
299 u->sessions_per_user_list_head_index,
300 oldest_per_user_translation_list_index);
301 /* Get the list element */
302 oldest_per_user_translation_list_elt =
303 pool_elt_at_index (sm->per_thread_data[thread_index].list_pool,
304 oldest_per_user_translation_list_index);
306 /* Get the session index from the list element */
307 session_index = oldest_per_user_translation_list_elt->value;
309 /* Get the session */
310 s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
312 } while (snat_is_session_static (s));
314 if (snat_is_unk_proto_session (s))
316 clib_bihash_kv_16_8_t up_kv;
317 snat_unk_proto_ses_key_t key;
319 /* Remove from lookup tables */
320 key.l_addr = s->in2out.addr;
321 key.r_addr = s->ext_host_addr;
322 key.fib_index = s->in2out.fib_index;
323 key.proto = s->in2out.port;
324 up_kv.key[0] = key.as_u64[0];
325 up_kv.key[1] = key.as_u64[1];
326 if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &up_kv, 0))
327 clib_warning ("in2out key del failed");
329 key.l_addr = s->out2in.addr;
330 key.fib_index = s->out2in.fib_index;
331 up_kv.key[0] = key.as_u64[0];
332 up_kv.key[1] = key.as_u64[1];
333 if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &up_kv, 0))
334 clib_warning ("out2in key del failed");
338 /* Remove in2out, out2in keys */
339 kv0.key = s->in2out.as_u64;
340 if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
341 clib_warning ("in2out key delete failed");
342 kv0.key = s->out2in.as_u64;
343 if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
344 clib_warning ("out2in key delete failed");
347 snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
348 s->out2in.addr.as_u32,
352 s->in2out.fib_index);
354 snat_free_outside_address_and_port
355 (sm, &s->out2in, s->outside_address_index);
357 s->outside_address_index = ~0;
359 if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
364 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
365 return SNAT_IN2OUT_NEXT_DROP;
367 s->outside_address_index = address_index;
371 u8 static_mapping = 1;
373 /* First try to match static mapping by local address and port */
374 if (snat_static_mapping_match (sm, *key0, &key1, 0, 0))
377 /* Try to create dynamic translation */
378 if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
381 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
382 return SNAT_IN2OUT_NEXT_DROP;
386 /* Create a new session */
387 pool_get (sm->per_thread_data[thread_index].sessions, s);
388 memset (s, 0, sizeof (*s));
390 s->outside_address_index = address_index;
394 u->nstaticsessions++;
395 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
402 /* Create list elts */
403 pool_get (sm->per_thread_data[thread_index].list_pool,
404 per_user_translation_list_elt);
405 clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
406 per_user_translation_list_elt -
407 sm->per_thread_data[thread_index].list_pool);
409 per_user_translation_list_elt->value =
410 s - sm->per_thread_data[thread_index].sessions;
411 s->per_user_index = per_user_translation_list_elt -
412 sm->per_thread_data[thread_index].list_pool;
413 s->per_user_list_head_index = u->sessions_per_user_list_head_index;
415 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
416 s->per_user_list_head_index,
417 per_user_translation_list_elt -
418 sm->per_thread_data[thread_index].list_pool);
423 s->out2in.protocol = key0->protocol;
424 s->out2in.fib_index = outside_fib_index;
425 s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
428 /* Add to translation hashes */
429 kv0.key = s->in2out.as_u64;
430 kv0.value = s - sm->per_thread_data[thread_index].sessions;
431 if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
432 clib_warning ("in2out key add failed");
434 kv0.key = s->out2in.as_u64;
435 kv0.value = s - sm->per_thread_data[thread_index].sessions;
437 if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
438 clib_warning ("out2in key add failed");
440 /* Add to translated packets worker lookup */
441 worker_by_out_key.addr = s->out2in.addr;
442 worker_by_out_key.port = s->out2in.port;
443 worker_by_out_key.fib_index = s->out2in.fib_index;
444 kv0.key = worker_by_out_key.as_u64;
445 kv0.value = thread_index;
446 clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1);
449 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
450 s->out2in.addr.as_u32,
454 s->in2out.fib_index);
459 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
460 snat_session_key_t *p_key0)
462 icmp46_header_t *icmp0;
463 snat_session_key_t key0;
464 icmp_echo_header_t *echo0, *inner_echo0 = 0;
465 ip4_header_t *inner_ip0 = 0;
467 icmp46_header_t *inner_icmp0;
469 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
470 echo0 = (icmp_echo_header_t *)(icmp0+1);
472 if (!icmp_is_error_message (icmp0))
474 key0.protocol = SNAT_PROTOCOL_ICMP;
475 key0.addr = ip0->src_address;
476 key0.port = echo0->identifier;
480 inner_ip0 = (ip4_header_t *)(echo0+1);
481 l4_header = ip4_next_header (inner_ip0);
482 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
483 key0.addr = inner_ip0->dst_address;
484 switch (key0.protocol)
486 case SNAT_PROTOCOL_ICMP:
487 inner_icmp0 = (icmp46_header_t*)l4_header;
488 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
489 key0.port = inner_echo0->identifier;
491 case SNAT_PROTOCOL_UDP:
492 case SNAT_PROTOCOL_TCP:
493 key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
496 return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
500 return -1; /* success */
504 * Get address and port values to be used for packet SNAT translation
505 * and create session if needed
507 * @param[in,out] sm SNAT main
508 * @param[in,out] node SNAT node runtime
509 * @param[in] thread_index thread index
510 * @param[in,out] b0 buffer containing packet to be translated
511 * @param[out] p_proto protocol used for matching
512 * @param[out] p_value address and port after NAT translation
513 * @param[out] p_dont_translate if packet should not be translated
514 * @param d optional parameter
515 * @param e optional parameter
517 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
518 u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
519 snat_session_key_t *p_value,
520 u8 *p_dont_translate, void *d, void *e)
523 icmp46_header_t *icmp0;
526 snat_session_key_t key0;
527 snat_session_t *s0 = 0;
528 u8 dont_translate = 0;
529 clib_bihash_kv_8_8_t kv0, value0;
534 if (PREDICT_FALSE(vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0))
536 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
538 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + iph_offset0);
539 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
540 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
541 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
543 err = icmp_get_key (ip0, &key0);
546 b0->error = node->errors[err];
547 next0 = SNAT_IN2OUT_NEXT_DROP;
550 key0.fib_index = rx_fib_index0;
552 kv0.key = key0.as_u64;
554 if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
556 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
557 IP_PROTOCOL_ICMP, rx_fib_index0) &&
558 vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
564 if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
566 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
567 next0 = SNAT_IN2OUT_NEXT_DROP;
571 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
572 &s0, node, next0, thread_index);
574 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
579 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
580 icmp0->type != ICMP4_echo_reply &&
581 !icmp_is_error_message (icmp0)))
583 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
584 next0 = SNAT_IN2OUT_NEXT_DROP;
588 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
593 *p_proto = key0.protocol;
595 *p_value = s0->out2in;
596 *p_dont_translate = dont_translate;
598 *(snat_session_t**)d = s0;
603 * Get address and port values to be used for packet SNAT translation
605 * @param[in] sm SNAT main
606 * @param[in,out] node SNAT node runtime
607 * @param[in] thread_index thread index
608 * @param[in,out] b0 buffer containing packet to be translated
609 * @param[out] p_proto protocol used for matching
610 * @param[out] p_value address and port after NAT translation
611 * @param[out] p_dont_translate if packet should not be translated
612 * @param d optional parameter
613 * @param e optional parameter
615 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
616 u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
617 snat_session_key_t *p_value,
618 u8 *p_dont_translate, void *d, void *e)
621 icmp46_header_t *icmp0;
624 snat_session_key_t key0;
625 snat_session_key_t sm0;
626 u8 dont_translate = 0;
631 ip0 = vlib_buffer_get_current (b0);
632 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
633 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
634 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
636 err = icmp_get_key (ip0, &key0);
639 b0->error = node->errors[err];
640 next0 = SNAT_IN2OUT_NEXT_DROP;
643 key0.fib_index = rx_fib_index0;
645 if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only))
647 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
648 IP_PROTOCOL_ICMP, rx_fib_index0)))
654 if (icmp_is_error_message (icmp0))
656 next0 = SNAT_IN2OUT_NEXT_DROP;
660 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
661 next0 = SNAT_IN2OUT_NEXT_DROP;
665 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
666 (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
667 !icmp_is_error_message (icmp0)))
669 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
670 next0 = SNAT_IN2OUT_NEXT_DROP;
677 *p_proto = key0.protocol;
678 *p_dont_translate = dont_translate;
682 static inline u32 icmp_in2out (snat_main_t *sm,
685 icmp46_header_t * icmp0,
688 vlib_node_runtime_t * node,
694 snat_session_key_t sm0;
696 icmp_echo_header_t *echo0, *inner_echo0 = 0;
697 ip4_header_t *inner_ip0;
699 icmp46_header_t *inner_icmp0;
701 u32 new_addr0, old_addr0;
702 u16 old_id0, new_id0;
707 echo0 = (icmp_echo_header_t *)(icmp0+1);
709 next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0,
710 &protocol, &sm0, &dont_translate, d, e);
713 if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
716 sum0 = ip_incremental_checksum (0, icmp0,
717 ntohs(ip0->length) - ip4_header_bytes (ip0));
718 checksum0 = ~ip_csum_fold (sum0);
719 if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
721 next0 = SNAT_IN2OUT_NEXT_DROP;
725 old_addr0 = ip0->src_address.as_u32;
726 new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
727 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
728 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
730 sum0 = ip0->checksum;
731 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
732 src_address /* changed member */);
733 ip0->checksum = ip_csum_fold (sum0);
735 if (!icmp_is_error_message (icmp0))
738 if (PREDICT_FALSE(new_id0 != echo0->identifier))
740 old_id0 = echo0->identifier;
742 echo0->identifier = new_id0;
744 sum0 = icmp0->checksum;
745 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
747 icmp0->checksum = ip_csum_fold (sum0);
752 inner_ip0 = (ip4_header_t *)(echo0+1);
753 l4_header = ip4_next_header (inner_ip0);
755 if (!ip4_header_checksum_is_valid (inner_ip0))
757 next0 = SNAT_IN2OUT_NEXT_DROP;
761 old_addr0 = inner_ip0->dst_address.as_u32;
762 inner_ip0->dst_address = sm0.addr;
763 new_addr0 = inner_ip0->dst_address.as_u32;
765 sum0 = icmp0->checksum;
766 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
767 dst_address /* changed member */);
768 icmp0->checksum = ip_csum_fold (sum0);
772 case SNAT_PROTOCOL_ICMP:
773 inner_icmp0 = (icmp46_header_t*)l4_header;
774 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
776 old_id0 = inner_echo0->identifier;
778 inner_echo0->identifier = new_id0;
780 sum0 = icmp0->checksum;
781 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
783 icmp0->checksum = ip_csum_fold (sum0);
785 case SNAT_PROTOCOL_UDP:
786 case SNAT_PROTOCOL_TCP:
787 old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
789 ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
791 sum0 = icmp0->checksum;
792 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
794 icmp0->checksum = ip_csum_fold (sum0);
808 * Hairpinning allows two endpoints on the internal side of the NAT to
809 * communicate even if they only use each other's external IP addresses
812 * @param sm SNAT main.
813 * @param b0 Vlib buffer.
814 * @param ip0 IP header.
815 * @param udp0 UDP header.
816 * @param tcp0 TCP header.
817 * @param proto0 SNAT protocol.
820 snat_hairpinning (snat_main_t *sm,
827 snat_session_key_t key0, sm0;
828 snat_worker_key_t k0;
830 clib_bihash_kv_8_8_t kv0, value0;
832 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
833 u16 new_dst_port0, old_dst_port0;
835 key0.addr = ip0->dst_address;
836 key0.port = udp0->dst_port;
837 key0.protocol = proto0;
838 key0.fib_index = sm->outside_fib_index;
839 kv0.key = key0.as_u64;
841 /* Check if destination is in active sessions */
842 if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
844 /* or static mappings */
845 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
847 new_dst_addr0 = sm0.addr.as_u32;
848 new_dst_port0 = sm0.port;
849 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
855 if (sm->num_workers > 1)
857 k0.addr = ip0->dst_address;
858 k0.port = udp0->dst_port;
859 k0.fib_index = sm->outside_fib_index;
861 if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
867 ti = sm->num_workers;
869 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
870 new_dst_addr0 = s0->in2out.addr.as_u32;
871 new_dst_port0 = s0->in2out.port;
872 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
875 /* Destination is behind the same NAT, use internal address and port */
878 old_dst_addr0 = ip0->dst_address.as_u32;
879 ip0->dst_address.as_u32 = new_dst_addr0;
880 sum0 = ip0->checksum;
881 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
882 ip4_header_t, dst_address);
883 ip0->checksum = ip_csum_fold (sum0);
885 old_dst_port0 = tcp0->dst;
886 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
888 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
890 tcp0->dst = new_dst_port0;
891 sum0 = tcp0->checksum;
892 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
893 ip4_header_t, dst_address);
894 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
895 ip4_header_t /* cheat */, length);
896 tcp0->checksum = ip_csum_fold(sum0);
900 udp0->dst_port = new_dst_port0;
906 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
908 sum0 = tcp0->checksum;
909 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
910 ip4_header_t, dst_address);
911 tcp0->checksum = ip_csum_fold(sum0);
918 snat_icmp_hairpinning (snat_main_t *sm,
921 icmp46_header_t * icmp0)
923 snat_session_key_t key0, sm0;
924 clib_bihash_kv_8_8_t kv0, value0;
925 snat_worker_key_t k0;
926 u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
930 if (!icmp_is_error_message (icmp0))
932 icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
933 u16 icmp_id0 = echo0->identifier;
934 key0.addr = ip0->dst_address;
935 key0.port = icmp_id0;
936 key0.protocol = SNAT_PROTOCOL_ICMP;
937 key0.fib_index = sm->outside_fib_index;
938 kv0.key = key0.as_u64;
940 /* Check if destination is in active sessions */
941 if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
943 /* or static mappings */
944 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0))
946 new_dst_addr0 = sm0.addr.as_u32;
947 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
953 if (sm->num_workers > 1)
955 k0.addr = ip0->dst_address;
957 k0.fib_index = sm->outside_fib_index;
959 if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv0, &value0))
965 ti = sm->num_workers;
967 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
968 new_dst_addr0 = s0->in2out.addr.as_u32;
969 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
970 echo0->identifier = s0->in2out.port;
971 sum0 = icmp0->checksum;
972 sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
973 icmp_echo_header_t, identifier);
974 icmp0->checksum = ip_csum_fold (sum0);
977 /* Destination is behind the same NAT, use internal address and port */
980 old_dst_addr0 = ip0->dst_address.as_u32;
981 ip0->dst_address.as_u32 = new_dst_addr0;
982 sum0 = ip0->checksum;
983 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
984 ip4_header_t, dst_address);
985 ip0->checksum = ip_csum_fold (sum0);
991 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
994 icmp46_header_t * icmp0,
997 vlib_node_runtime_t * node,
1001 snat_session_t ** p_s0)
1003 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1004 next0, thread_index, p_s0, 0);
1005 snat_session_t * s0 = *p_s0;
1006 if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1009 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1010 snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1012 s0->last_heard = now;
1014 s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1015 /* Per-user LRU list maintenance for dynamic translations */
1016 if (!snat_is_session_static (s0))
1018 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1019 s0->per_user_index);
1020 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1021 s0->per_user_list_head_index,
1022 s0->per_user_index);
1028 snat_hairpinning_unknown_proto (snat_main_t *sm,
1032 u32 old_addr, new_addr = 0, ti = 0;
1033 clib_bihash_kv_8_8_t kv, value;
1034 clib_bihash_kv_16_8_t s_kv, s_value;
1035 snat_unk_proto_ses_key_t key;
1036 snat_session_key_t m_key;
1037 snat_worker_key_t w_key;
1038 snat_static_mapping_t *m;
1042 old_addr = ip->dst_address.as_u32;
1043 key.l_addr.as_u32 = ip->dst_address.as_u32;
1044 key.r_addr.as_u32 = ip->src_address.as_u32;
1045 key.fib_index = sm->outside_fib_index;
1046 key.proto = ip->protocol;
1047 key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
1048 s_kv.key[0] = key.as_u64[0];
1049 s_kv.key[1] = key.as_u64[1];
1050 if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
1052 m_key.addr = ip->dst_address;
1053 m_key.fib_index = sm->outside_fib_index;
1056 kv.key = m_key.as_u64;
1057 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1060 m = pool_elt_at_index (sm->static_mappings, value.value);
1061 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1062 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1063 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1067 if (sm->num_workers > 1)
1069 w_key.addr = ip->dst_address;
1071 w_key.fib_index = sm->outside_fib_index;
1072 kv.key = w_key.as_u64;
1073 if (clib_bihash_search_8_8 (&sm->worker_by_out, &kv, &value))
1079 ti = sm->num_workers;
1081 s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1082 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1083 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1084 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1087 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1088 ip->checksum = ip_csum_fold (sum);
1092 snat_in2out_unknown_proto (snat_main_t *sm,
1100 clib_bihash_kv_8_8_t kv, value;
1101 clib_bihash_kv_16_8_t s_kv, s_value;
1102 snat_static_mapping_t *m;
1103 snat_session_key_t m_key;
1104 u32 old_addr, new_addr = 0;
1106 snat_user_key_t u_key;
1108 dlist_elt_t *head, *elt, *oldest;
1109 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1110 u32 elt_index, head_index, ses_index, oldest_index;
1112 snat_unk_proto_ses_key_t key;
1113 u32 address_index = ~0;
1117 old_addr = ip->src_address.as_u32;
1119 key.l_addr = ip->src_address;
1120 key.r_addr = ip->dst_address;
1121 key.fib_index = rx_fib_index;
1122 key.proto = ip->protocol;
1123 key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
1124 s_kv.key[0] = key.as_u64[0];
1125 s_kv.key[1] = key.as_u64[1];
1127 if (!clib_bihash_search_16_8 (&sm->in2out_unk_proto, &s_kv, &s_value))
1129 s = pool_elt_at_index (tsm->sessions, s_value.value);
1130 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1134 u_key.addr = ip->src_address;
1135 u_key.fib_index = rx_fib_index;
1136 kv.key = u_key.as_u64;
1138 /* Ever heard of the "user" = src ip4 address before? */
1139 if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
1141 /* no, make a new one */
1142 pool_get (tsm->users, u);
1143 memset (u, 0, sizeof (*u));
1144 u->addr = ip->src_address;
1145 u->fib_index = rx_fib_index;
1147 pool_get (tsm->list_pool, head);
1148 u->sessions_per_user_list_head_index = head - tsm->list_pool;
1150 clib_dlist_init (tsm->list_pool,
1151 u->sessions_per_user_list_head_index);
1153 kv.value = u - tsm->users;
1156 clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
1160 u = pool_elt_at_index (tsm->users, value.value);
1163 m_key.addr = ip->src_address;
1166 m_key.fib_index = rx_fib_index;
1167 kv.key = m_key.as_u64;
1169 /* Try to find static mapping first */
1170 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1172 m = pool_elt_at_index (sm->static_mappings, value.value);
1173 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1177 /* Fallback to 3-tuple key */
1180 /* Choose same out address as for TCP/UDP session to same destination */
1181 if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
1183 head_index = u->sessions_per_user_list_head_index;
1184 head = pool_elt_at_index (tsm->list_pool, head_index);
1185 elt_index = head->next;
1186 elt = pool_elt_at_index (tsm->list_pool, elt_index);
1187 ses_index = elt->value;
1188 while (ses_index != ~0)
1190 s = pool_elt_at_index (tsm->sessions, ses_index);
1191 elt_index = elt->next;
1192 elt = pool_elt_at_index (tsm->list_pool, elt_index);
1193 ses_index = elt->value;
1195 if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1197 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1198 address_index = s->outside_address_index;
1200 key.fib_index = sm->outside_fib_index;
1201 key.l_addr.as_u32 = new_addr;
1202 s_kv.key[0] = key.as_u64[0];
1203 s_kv.key[1] = key.as_u64[1];
1204 if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
1211 key.fib_index = sm->outside_fib_index;
1212 for (i = 0; i < vec_len (sm->addresses); i++)
1214 key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1215 s_kv.key[0] = key.as_u64[0];
1216 s_kv.key[1] = key.as_u64[1];
1217 if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
1219 new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1228 /* Over quota? Recycle the least recently used dynamic translation */
1229 if (u->nsessions >= sm->max_translations_per_user && !is_sm)
1231 /* Remove the oldest dynamic translation */
1233 oldest_index = clib_dlist_remove_head (
1234 tsm->list_pool, u->sessions_per_user_list_head_index);
1236 ASSERT (oldest_index != ~0);
1238 /* add it back to the end of the LRU list */
1239 clib_dlist_addtail (tsm->list_pool,
1240 u->sessions_per_user_list_head_index,
1242 /* Get the list element */
1243 oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
1245 /* Get the session index from the list element */
1246 ses_index = oldest->value;
1248 /* Get the session */
1249 s = pool_elt_at_index (tsm->sessions, ses_index);
1250 } while (snat_is_session_static (s));
1252 if (snat_is_unk_proto_session (s))
1254 /* Remove from lookup tables */
1255 key.l_addr = s->in2out.addr;
1256 key.r_addr = s->ext_host_addr;
1257 key.fib_index = s->in2out.fib_index;
1258 key.proto = s->in2out.port;
1259 s_kv.key[0] = key.as_u64[0];
1260 s_kv.key[1] = key.as_u64[1];
1261 if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 0))
1262 clib_warning ("in2out key del failed");
1264 key.l_addr = s->out2in.addr;
1265 key.fib_index = s->out2in.fib_index;
1266 s_kv.key[0] = key.as_u64[0];
1267 s_kv.key[1] = key.as_u64[1];
1268 if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 0))
1269 clib_warning ("out2in key del failed");
1274 snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
1275 s->out2in.addr.as_u32,
1279 s->in2out.fib_index);
1281 snat_free_outside_address_and_port (sm, &s->out2in,
1282 s->outside_address_index);
1284 /* Remove in2out, out2in keys */
1285 kv.key = s->in2out.as_u64;
1286 if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
1287 clib_warning ("in2out key del failed");
1288 kv.key = s->out2in.as_u64;
1289 if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
1290 clib_warning ("out2in key del failed");
1295 /* Create a new session */
1296 pool_get (tsm->sessions, s);
1297 memset (s, 0, sizeof (*s));
1299 /* Create list elts */
1300 pool_get (tsm->list_pool, elt);
1301 clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
1302 elt->value = s - tsm->sessions;
1303 s->per_user_index = elt - tsm->list_pool;
1304 s->per_user_list_head_index = u->sessions_per_user_list_head_index;
1305 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1309 s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1310 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1311 s->outside_address_index = address_index;
1312 s->out2in.addr.as_u32 = new_addr;
1313 s->out2in.fib_index = sm->outside_fib_index;
1314 s->in2out.addr.as_u32 = old_addr;
1315 s->in2out.fib_index = rx_fib_index;
1316 s->in2out.port = s->out2in.port = ip->protocol;
1319 u->nstaticsessions++;
1320 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1327 /* Add to lookup tables */
1328 key.l_addr.as_u32 = old_addr;
1329 key.r_addr = ip->dst_address;
1330 key.proto = ip->protocol;
1331 key.fib_index = rx_fib_index;
1332 s_kv.key[0] = key.as_u64[0];
1333 s_kv.key[1] = key.as_u64[1];
1334 s_kv.value = s - tsm->sessions;
1335 if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 1))
1336 clib_warning ("in2out key add failed");
1338 key.l_addr.as_u32 = new_addr;
1339 key.fib_index = sm->outside_fib_index;
1340 s_kv.key[0] = key.as_u64[0];
1341 s_kv.key[1] = key.as_u64[1];
1342 if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 1))
1343 clib_warning ("out2in key add failed");
1346 /* Update IP checksum */
1348 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1349 ip->checksum = ip_csum_fold (sum);
1352 s->last_heard = now;
1354 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1355 /* Per-user LRU list maintenance */
1356 clib_dlist_remove (tsm->list_pool, s->per_user_index);
1357 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1361 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1362 snat_hairpinning_unknown_proto(sm, b, ip);
1364 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1365 vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1369 snat_in2out_node_fn_inline (vlib_main_t * vm,
1370 vlib_node_runtime_t * node,
1371 vlib_frame_t * frame, int is_slow_path,
1372 int is_output_feature)
1374 u32 n_left_from, * from, * to_next;
1375 snat_in2out_next_t next_index;
1376 u32 pkts_processed = 0;
1377 snat_main_t * sm = &snat_main;
1378 f64 now = vlib_time_now (vm);
1379 u32 stats_node_index;
1380 u32 thread_index = vlib_get_thread_index ();
1382 stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1383 snat_in2out_node.index;
1385 from = vlib_frame_vector_args (frame);
1386 n_left_from = frame->n_vectors;
1387 next_index = node->cached_next_index;
1389 while (n_left_from > 0)
1393 vlib_get_next_frame (vm, node, next_index,
1394 to_next, n_left_to_next);
1396 while (n_left_from >= 4 && n_left_to_next >= 2)
1399 vlib_buffer_t * b0, * b1;
1401 u32 sw_if_index0, sw_if_index1;
1402 ip4_header_t * ip0, * ip1;
1403 ip_csum_t sum0, sum1;
1404 u32 new_addr0, old_addr0, new_addr1, old_addr1;
1405 u16 old_port0, new_port0, old_port1, new_port1;
1406 udp_header_t * udp0, * udp1;
1407 tcp_header_t * tcp0, * tcp1;
1408 icmp46_header_t * icmp0, * icmp1;
1409 snat_session_key_t key0, key1;
1410 u32 rx_fib_index0, rx_fib_index1;
1412 snat_session_t * s0 = 0, * s1 = 0;
1413 clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1414 u32 iph_offset0 = 0, iph_offset1 = 0;
1416 /* Prefetch next iteration. */
1418 vlib_buffer_t * p2, * p3;
1420 p2 = vlib_get_buffer (vm, from[2]);
1421 p3 = vlib_get_buffer (vm, from[3]);
1423 vlib_prefetch_buffer_header (p2, LOAD);
1424 vlib_prefetch_buffer_header (p3, LOAD);
1426 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1427 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1430 /* speculatively enqueue b0 and b1 to the current next frame */
1431 to_next[0] = bi0 = from[0];
1432 to_next[1] = bi1 = from[1];
1436 n_left_to_next -= 2;
1438 b0 = vlib_get_buffer (vm, bi0);
1439 b1 = vlib_get_buffer (vm, bi1);
1441 if (is_output_feature)
1442 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1444 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1447 udp0 = ip4_next_header (ip0);
1448 tcp0 = (tcp_header_t *) udp0;
1449 icmp0 = (icmp46_header_t *) udp0;
1451 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1452 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1455 next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1457 if (PREDICT_FALSE(ip0->ttl == 1))
1459 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1460 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1461 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1463 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1467 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1469 /* Next configured feature, probably ip4-lookup */
1472 if (PREDICT_FALSE (proto0 == ~0))
1474 snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1475 thread_index, now, vm);
1479 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1481 next0 = icmp_in2out_slow_path
1482 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1483 node, next0, now, thread_index, &s0);
1489 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1491 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1496 key0.addr = ip0->src_address;
1497 key0.port = udp0->src_port;
1498 key0.protocol = proto0;
1499 key0.fib_index = rx_fib_index0;
1501 kv0.key = key0.as_u64;
1503 if (PREDICT_FALSE (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0) != 0))
1507 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1508 ip0, proto0, rx_fib_index0)) && !is_output_feature)
1511 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1512 &s0, node, next0, thread_index);
1513 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1518 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1523 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1526 old_addr0 = ip0->src_address.as_u32;
1527 ip0->src_address = s0->out2in.addr;
1528 new_addr0 = ip0->src_address.as_u32;
1529 if (!is_output_feature)
1530 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1532 sum0 = ip0->checksum;
1533 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1535 src_address /* changed member */);
1536 ip0->checksum = ip_csum_fold (sum0);
1538 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1540 old_port0 = tcp0->src_port;
1541 tcp0->src_port = s0->out2in.port;
1542 new_port0 = tcp0->src_port;
1544 sum0 = tcp0->checksum;
1545 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1547 dst_address /* changed member */);
1548 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1549 ip4_header_t /* cheat */,
1550 length /* changed member */);
1551 tcp0->checksum = ip_csum_fold(sum0);
1555 old_port0 = udp0->src_port;
1556 udp0->src_port = s0->out2in.port;
1561 if (!is_output_feature)
1562 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1565 s0->last_heard = now;
1567 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1568 /* Per-user LRU list maintenance for dynamic translation */
1569 if (!snat_is_session_static (s0))
1571 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1572 s0->per_user_index);
1573 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1574 s0->per_user_list_head_index,
1575 s0->per_user_index);
1579 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1580 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1582 snat_in2out_trace_t *t =
1583 vlib_add_trace (vm, node, b0, sizeof (*t));
1584 t->is_slow_path = is_slow_path;
1585 t->sw_if_index = sw_if_index0;
1586 t->next_index = next0;
1587 t->session_index = ~0;
1589 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1592 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1594 if (is_output_feature)
1595 iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1597 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1600 udp1 = ip4_next_header (ip1);
1601 tcp1 = (tcp_header_t *) udp1;
1602 icmp1 = (icmp46_header_t *) udp1;
1604 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1605 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1608 if (PREDICT_FALSE(ip1->ttl == 1))
1610 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1611 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1612 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1614 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1618 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1620 /* Next configured feature, probably ip4-lookup */
1623 if (PREDICT_FALSE (proto1 == ~0))
1625 snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1626 thread_index, now, vm);
1630 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1632 next1 = icmp_in2out_slow_path
1633 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1634 next1, now, thread_index, &s1);
1640 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1642 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1647 key1.addr = ip1->src_address;
1648 key1.port = udp1->src_port;
1649 key1.protocol = proto1;
1650 key1.fib_index = rx_fib_index1;
1652 kv1.key = key1.as_u64;
1654 if (PREDICT_FALSE(clib_bihash_search_8_8 (&sm->in2out, &kv1, &value1) != 0))
1658 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1659 ip1, proto1, rx_fib_index1)) && !is_output_feature)
1662 next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1663 &s1, node, next1, thread_index);
1664 if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1669 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1674 s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1677 old_addr1 = ip1->src_address.as_u32;
1678 ip1->src_address = s1->out2in.addr;
1679 new_addr1 = ip1->src_address.as_u32;
1680 if (!is_output_feature)
1681 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1683 sum1 = ip1->checksum;
1684 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1686 src_address /* changed member */);
1687 ip1->checksum = ip_csum_fold (sum1);
1689 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1691 old_port1 = tcp1->src_port;
1692 tcp1->src_port = s1->out2in.port;
1693 new_port1 = tcp1->src_port;
1695 sum1 = tcp1->checksum;
1696 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1698 dst_address /* changed member */);
1699 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1700 ip4_header_t /* cheat */,
1701 length /* changed member */);
1702 tcp1->checksum = ip_csum_fold(sum1);
1706 old_port1 = udp1->src_port;
1707 udp1->src_port = s1->out2in.port;
1712 if (!is_output_feature)
1713 snat_hairpinning (sm, b1, ip1, udp1, tcp1, proto1);
1716 s1->last_heard = now;
1718 s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1719 /* Per-user LRU list maintenance for dynamic translation */
1720 if (!snat_is_session_static (s1))
1722 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1723 s1->per_user_index);
1724 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1725 s1->per_user_list_head_index,
1726 s1->per_user_index);
1730 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1731 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1733 snat_in2out_trace_t *t =
1734 vlib_add_trace (vm, node, b1, sizeof (*t));
1735 t->sw_if_index = sw_if_index1;
1736 t->next_index = next1;
1737 t->session_index = ~0;
1739 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1742 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1744 /* verify speculative enqueues, maybe switch current next frame */
1745 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1746 to_next, n_left_to_next,
1747 bi0, bi1, next0, next1);
1750 while (n_left_from > 0 && n_left_to_next > 0)
1758 u32 new_addr0, old_addr0;
1759 u16 old_port0, new_port0;
1760 udp_header_t * udp0;
1761 tcp_header_t * tcp0;
1762 icmp46_header_t * icmp0;
1763 snat_session_key_t key0;
1766 snat_session_t * s0 = 0;
1767 clib_bihash_kv_8_8_t kv0, value0;
1768 u32 iph_offset0 = 0;
1770 /* speculatively enqueue b0 to the current next frame */
1776 n_left_to_next -= 1;
1778 b0 = vlib_get_buffer (vm, bi0);
1779 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1781 if (is_output_feature)
1782 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1784 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1787 udp0 = ip4_next_header (ip0);
1788 tcp0 = (tcp_header_t *) udp0;
1789 icmp0 = (icmp46_header_t *) udp0;
1791 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1792 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1795 if (PREDICT_FALSE(ip0->ttl == 1))
1797 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1798 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1799 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1801 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1805 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1807 /* Next configured feature, probably ip4-lookup */
1810 if (PREDICT_FALSE (proto0 == ~0))
1812 snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1813 thread_index, now, vm);
1817 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1819 next0 = icmp_in2out_slow_path
1820 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1821 next0, now, thread_index, &s0);
1827 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1829 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1834 key0.addr = ip0->src_address;
1835 key0.port = udp0->src_port;
1836 key0.protocol = proto0;
1837 key0.fib_index = rx_fib_index0;
1839 kv0.key = key0.as_u64;
1841 if (clib_bihash_search_8_8 (&sm->in2out, &kv0, &value0))
1845 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1846 ip0, proto0, rx_fib_index0)) && !is_output_feature)
1849 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1850 &s0, node, next0, thread_index);
1852 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1857 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1862 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1865 old_addr0 = ip0->src_address.as_u32;
1866 ip0->src_address = s0->out2in.addr;
1867 new_addr0 = ip0->src_address.as_u32;
1868 if (!is_output_feature)
1869 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1871 sum0 = ip0->checksum;
1872 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1874 src_address /* changed member */);
1875 ip0->checksum = ip_csum_fold (sum0);
1877 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1879 old_port0 = tcp0->src_port;
1880 tcp0->src_port = s0->out2in.port;
1881 new_port0 = tcp0->src_port;
1883 sum0 = tcp0->checksum;
1884 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1886 dst_address /* changed member */);
1887 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1888 ip4_header_t /* cheat */,
1889 length /* changed member */);
1890 tcp0->checksum = ip_csum_fold(sum0);
1894 old_port0 = udp0->src_port;
1895 udp0->src_port = s0->out2in.port;
1900 if (!is_output_feature)
1901 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
1904 s0->last_heard = now;
1906 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1907 /* Per-user LRU list maintenance for dynamic translation */
1908 if (!snat_is_session_static (s0))
1910 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1911 s0->per_user_index);
1912 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1913 s0->per_user_list_head_index,
1914 s0->per_user_index);
1918 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1919 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1921 snat_in2out_trace_t *t =
1922 vlib_add_trace (vm, node, b0, sizeof (*t));
1923 t->is_slow_path = is_slow_path;
1924 t->sw_if_index = sw_if_index0;
1925 t->next_index = next0;
1926 t->session_index = ~0;
1928 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1931 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1933 /* verify speculative enqueue, maybe switch current next frame */
1934 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1935 to_next, n_left_to_next,
1939 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1942 vlib_node_increment_counter (vm, stats_node_index,
1943 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1945 return frame->n_vectors;
1949 snat_in2out_fast_path_fn (vlib_main_t * vm,
1950 vlib_node_runtime_t * node,
1951 vlib_frame_t * frame)
1953 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1956 VLIB_REGISTER_NODE (snat_in2out_node) = {
1957 .function = snat_in2out_fast_path_fn,
1958 .name = "snat-in2out",
1959 .vector_size = sizeof (u32),
1960 .format_trace = format_snat_in2out_trace,
1961 .type = VLIB_NODE_TYPE_INTERNAL,
1963 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1964 .error_strings = snat_in2out_error_strings,
1966 .runtime_data_bytes = sizeof (snat_runtime_t),
1968 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1970 /* edit / add dispositions here */
1972 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1973 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1974 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
1975 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1979 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1982 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1983 vlib_node_runtime_t * node,
1984 vlib_frame_t * frame)
1986 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1989 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1990 .function = snat_in2out_output_fast_path_fn,
1991 .name = "snat-in2out-output",
1992 .vector_size = sizeof (u32),
1993 .format_trace = format_snat_in2out_trace,
1994 .type = VLIB_NODE_TYPE_INTERNAL,
1996 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1997 .error_strings = snat_in2out_error_strings,
1999 .runtime_data_bytes = sizeof (snat_runtime_t),
2001 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2003 /* edit / add dispositions here */
2005 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2006 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2007 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-output-slowpath",
2008 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2012 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2013 snat_in2out_output_fast_path_fn);
2016 snat_in2out_slow_path_fn (vlib_main_t * vm,
2017 vlib_node_runtime_t * node,
2018 vlib_frame_t * frame)
2020 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2023 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2024 .function = snat_in2out_slow_path_fn,
2025 .name = "snat-in2out-slowpath",
2026 .vector_size = sizeof (u32),
2027 .format_trace = format_snat_in2out_trace,
2028 .type = VLIB_NODE_TYPE_INTERNAL,
2030 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2031 .error_strings = snat_in2out_error_strings,
2033 .runtime_data_bytes = sizeof (snat_runtime_t),
2035 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2037 /* edit / add dispositions here */
2039 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2040 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2041 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
2042 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2046 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2047 snat_in2out_slow_path_fn);
2050 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2051 vlib_node_runtime_t * node,
2052 vlib_frame_t * frame)
2054 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2057 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2058 .function = snat_in2out_output_slow_path_fn,
2059 .name = "snat-in2out-output-slowpath",
2060 .vector_size = sizeof (u32),
2061 .format_trace = format_snat_in2out_trace,
2062 .type = VLIB_NODE_TYPE_INTERNAL,
2064 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2065 .error_strings = snat_in2out_error_strings,
2067 .runtime_data_bytes = sizeof (snat_runtime_t),
2069 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2071 /* edit / add dispositions here */
2073 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2074 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2075 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-output-slowpath",
2076 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2080 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2081 snat_in2out_output_slow_path_fn);
2083 /**************************/
2084 /*** deterministic mode ***/
2085 /**************************/
2087 snat_det_in2out_node_fn (vlib_main_t * vm,
2088 vlib_node_runtime_t * node,
2089 vlib_frame_t * frame)
2091 u32 n_left_from, * from, * to_next;
2092 snat_in2out_next_t next_index;
2093 u32 pkts_processed = 0;
2094 snat_main_t * sm = &snat_main;
2095 u32 now = (u32) vlib_time_now (vm);
2096 u32 thread_index = vlib_get_thread_index ();
2098 from = vlib_frame_vector_args (frame);
2099 n_left_from = frame->n_vectors;
2100 next_index = node->cached_next_index;
2102 while (n_left_from > 0)
2106 vlib_get_next_frame (vm, node, next_index,
2107 to_next, n_left_to_next);
2109 while (n_left_from >= 4 && n_left_to_next >= 2)
2112 vlib_buffer_t * b0, * b1;
2114 u32 sw_if_index0, sw_if_index1;
2115 ip4_header_t * ip0, * ip1;
2116 ip_csum_t sum0, sum1;
2117 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2118 u16 old_port0, new_port0, lo_port0, i0;
2119 u16 old_port1, new_port1, lo_port1, i1;
2120 udp_header_t * udp0, * udp1;
2121 tcp_header_t * tcp0, * tcp1;
2123 snat_det_out_key_t key0, key1;
2124 snat_det_map_t * dm0, * dm1;
2125 snat_det_session_t * ses0 = 0, * ses1 = 0;
2126 u32 rx_fib_index0, rx_fib_index1;
2127 icmp46_header_t * icmp0, * icmp1;
2129 /* Prefetch next iteration. */
2131 vlib_buffer_t * p2, * p3;
2133 p2 = vlib_get_buffer (vm, from[2]);
2134 p3 = vlib_get_buffer (vm, from[3]);
2136 vlib_prefetch_buffer_header (p2, LOAD);
2137 vlib_prefetch_buffer_header (p3, LOAD);
2139 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2140 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2143 /* speculatively enqueue b0 and b1 to the current next frame */
2144 to_next[0] = bi0 = from[0];
2145 to_next[1] = bi1 = from[1];
2149 n_left_to_next -= 2;
2151 b0 = vlib_get_buffer (vm, bi0);
2152 b1 = vlib_get_buffer (vm, bi1);
2154 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2155 next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2157 ip0 = vlib_buffer_get_current (b0);
2158 udp0 = ip4_next_header (ip0);
2159 tcp0 = (tcp_header_t *) udp0;
2161 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2163 if (PREDICT_FALSE(ip0->ttl == 1))
2165 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2166 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2167 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2169 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2173 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2175 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2177 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2178 icmp0 = (icmp46_header_t *) udp0;
2180 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2181 rx_fib_index0, node, next0, thread_index,
2186 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2187 if (PREDICT_FALSE(!dm0))
2189 clib_warning("no match for internal host %U",
2190 format_ip4_address, &ip0->src_address);
2191 next0 = SNAT_IN2OUT_NEXT_DROP;
2192 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2196 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2198 key0.ext_host_addr = ip0->dst_address;
2199 key0.ext_host_port = tcp0->dst;
2201 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2202 if (PREDICT_FALSE(!ses0))
2204 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2206 key0.out_port = clib_host_to_net_u16 (lo_port0 +
2207 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2209 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2212 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2215 if (PREDICT_FALSE(!ses0))
2217 /* too many sessions for user, send ICMP error packet */
2219 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2220 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2221 ICMP4_destination_unreachable_destination_unreachable_host,
2223 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2228 new_port0 = ses0->out.out_port;
2230 old_addr0.as_u32 = ip0->src_address.as_u32;
2231 ip0->src_address.as_u32 = new_addr0.as_u32;
2232 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2234 sum0 = ip0->checksum;
2235 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2237 src_address /* changed member */);
2238 ip0->checksum = ip_csum_fold (sum0);
2240 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2242 if (tcp0->flags & TCP_FLAG_SYN)
2243 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2244 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2245 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2246 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2247 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2248 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2249 snat_det_ses_close(dm0, ses0);
2250 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2251 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2252 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2253 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2255 old_port0 = tcp0->src;
2256 tcp0->src = new_port0;
2258 sum0 = tcp0->checksum;
2259 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2261 dst_address /* changed member */);
2262 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2263 ip4_header_t /* cheat */,
2264 length /* changed member */);
2265 tcp0->checksum = ip_csum_fold(sum0);
2269 ses0->state = SNAT_SESSION_UDP_ACTIVE;
2270 old_port0 = udp0->src_port;
2271 udp0->src_port = new_port0;
2277 case SNAT_SESSION_UDP_ACTIVE:
2278 ses0->expire = now + sm->udp_timeout;
2280 case SNAT_SESSION_TCP_SYN_SENT:
2281 case SNAT_SESSION_TCP_FIN_WAIT:
2282 case SNAT_SESSION_TCP_CLOSE_WAIT:
2283 case SNAT_SESSION_TCP_LAST_ACK:
2284 ses0->expire = now + sm->tcp_transitory_timeout;
2286 case SNAT_SESSION_TCP_ESTABLISHED:
2287 ses0->expire = now + sm->tcp_established_timeout;
2292 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2293 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2295 snat_in2out_trace_t *t =
2296 vlib_add_trace (vm, node, b0, sizeof (*t));
2297 t->is_slow_path = 0;
2298 t->sw_if_index = sw_if_index0;
2299 t->next_index = next0;
2300 t->session_index = ~0;
2302 t->session_index = ses0 - dm0->sessions;
2305 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2307 ip1 = vlib_buffer_get_current (b1);
2308 udp1 = ip4_next_header (ip1);
2309 tcp1 = (tcp_header_t *) udp1;
2311 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2313 if (PREDICT_FALSE(ip1->ttl == 1))
2315 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2316 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2317 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2319 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2323 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2325 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2327 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2328 icmp1 = (icmp46_header_t *) udp1;
2330 next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2331 rx_fib_index1, node, next1, thread_index,
2336 dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2337 if (PREDICT_FALSE(!dm1))
2339 clib_warning("no match for internal host %U",
2340 format_ip4_address, &ip0->src_address);
2341 next1 = SNAT_IN2OUT_NEXT_DROP;
2342 b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2346 snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
2348 key1.ext_host_addr = ip1->dst_address;
2349 key1.ext_host_port = tcp1->dst;
2351 ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
2352 if (PREDICT_FALSE(!ses1))
2354 for (i1 = 0; i1 < dm1->ports_per_host; i1++)
2356 key1.out_port = clib_host_to_net_u16 (lo_port1 +
2357 ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
2359 if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
2362 ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
2365 if (PREDICT_FALSE(!ses1))
2367 /* too many sessions for user, send ICMP error packet */
2369 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2370 icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
2371 ICMP4_destination_unreachable_destination_unreachable_host,
2373 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2378 new_port1 = ses1->out.out_port;
2380 old_addr1.as_u32 = ip1->src_address.as_u32;
2381 ip1->src_address.as_u32 = new_addr1.as_u32;
2382 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2384 sum1 = ip1->checksum;
2385 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2387 src_address /* changed member */);
2388 ip1->checksum = ip_csum_fold (sum1);
2390 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2392 if (tcp1->flags & TCP_FLAG_SYN)
2393 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
2394 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
2395 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2396 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2397 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
2398 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
2399 snat_det_ses_close(dm1, ses1);
2400 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2401 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
2402 else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
2403 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2405 old_port1 = tcp1->src;
2406 tcp1->src = new_port1;
2408 sum1 = tcp1->checksum;
2409 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2411 dst_address /* changed member */);
2412 sum1 = ip_csum_update (sum1, old_port1, new_port1,
2413 ip4_header_t /* cheat */,
2414 length /* changed member */);
2415 tcp1->checksum = ip_csum_fold(sum1);
2419 ses1->state = SNAT_SESSION_UDP_ACTIVE;
2420 old_port1 = udp1->src_port;
2421 udp1->src_port = new_port1;
2427 case SNAT_SESSION_UDP_ACTIVE:
2428 ses1->expire = now + sm->udp_timeout;
2430 case SNAT_SESSION_TCP_SYN_SENT:
2431 case SNAT_SESSION_TCP_FIN_WAIT:
2432 case SNAT_SESSION_TCP_CLOSE_WAIT:
2433 case SNAT_SESSION_TCP_LAST_ACK:
2434 ses1->expire = now + sm->tcp_transitory_timeout;
2436 case SNAT_SESSION_TCP_ESTABLISHED:
2437 ses1->expire = now + sm->tcp_established_timeout;
2442 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2443 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2445 snat_in2out_trace_t *t =
2446 vlib_add_trace (vm, node, b1, sizeof (*t));
2447 t->is_slow_path = 0;
2448 t->sw_if_index = sw_if_index1;
2449 t->next_index = next1;
2450 t->session_index = ~0;
2452 t->session_index = ses1 - dm1->sessions;
2455 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
2457 /* verify speculative enqueues, maybe switch current next frame */
2458 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2459 to_next, n_left_to_next,
2460 bi0, bi1, next0, next1);
2463 while (n_left_from > 0 && n_left_to_next > 0)
2471 ip4_address_t new_addr0, old_addr0;
2472 u16 old_port0, new_port0, lo_port0, i0;
2473 udp_header_t * udp0;
2474 tcp_header_t * tcp0;
2476 snat_det_out_key_t key0;
2477 snat_det_map_t * dm0;
2478 snat_det_session_t * ses0 = 0;
2480 icmp46_header_t * icmp0;
2482 /* speculatively enqueue b0 to the current next frame */
2488 n_left_to_next -= 1;
2490 b0 = vlib_get_buffer (vm, bi0);
2491 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2493 ip0 = vlib_buffer_get_current (b0);
2494 udp0 = ip4_next_header (ip0);
2495 tcp0 = (tcp_header_t *) udp0;
2497 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2499 if (PREDICT_FALSE(ip0->ttl == 1))
2501 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2502 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2503 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2505 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2509 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2511 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2513 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2514 icmp0 = (icmp46_header_t *) udp0;
2516 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2517 rx_fib_index0, node, next0, thread_index,
2522 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2523 if (PREDICT_FALSE(!dm0))
2525 clib_warning("no match for internal host %U",
2526 format_ip4_address, &ip0->src_address);
2527 next0 = SNAT_IN2OUT_NEXT_DROP;
2528 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2532 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2534 key0.ext_host_addr = ip0->dst_address;
2535 key0.ext_host_port = tcp0->dst;
2537 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2538 if (PREDICT_FALSE(!ses0))
2540 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2542 key0.out_port = clib_host_to_net_u16 (lo_port0 +
2543 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2545 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2548 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2551 if (PREDICT_FALSE(!ses0))
2553 /* too many sessions for user, send ICMP error packet */
2555 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2556 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2557 ICMP4_destination_unreachable_destination_unreachable_host,
2559 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2564 new_port0 = ses0->out.out_port;
2566 old_addr0.as_u32 = ip0->src_address.as_u32;
2567 ip0->src_address.as_u32 = new_addr0.as_u32;
2568 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2570 sum0 = ip0->checksum;
2571 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2573 src_address /* changed member */);
2574 ip0->checksum = ip_csum_fold (sum0);
2576 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2578 if (tcp0->flags & TCP_FLAG_SYN)
2579 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2580 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2581 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2582 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2583 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2584 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2585 snat_det_ses_close(dm0, ses0);
2586 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2587 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2588 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2589 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2591 old_port0 = tcp0->src;
2592 tcp0->src = new_port0;
2594 sum0 = tcp0->checksum;
2595 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2597 dst_address /* changed member */);
2598 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2599 ip4_header_t /* cheat */,
2600 length /* changed member */);
2601 tcp0->checksum = ip_csum_fold(sum0);
2605 ses0->state = SNAT_SESSION_UDP_ACTIVE;
2606 old_port0 = udp0->src_port;
2607 udp0->src_port = new_port0;
2613 case SNAT_SESSION_UDP_ACTIVE:
2614 ses0->expire = now + sm->udp_timeout;
2616 case SNAT_SESSION_TCP_SYN_SENT:
2617 case SNAT_SESSION_TCP_FIN_WAIT:
2618 case SNAT_SESSION_TCP_CLOSE_WAIT:
2619 case SNAT_SESSION_TCP_LAST_ACK:
2620 ses0->expire = now + sm->tcp_transitory_timeout;
2622 case SNAT_SESSION_TCP_ESTABLISHED:
2623 ses0->expire = now + sm->tcp_established_timeout;
2628 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2629 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2631 snat_in2out_trace_t *t =
2632 vlib_add_trace (vm, node, b0, sizeof (*t));
2633 t->is_slow_path = 0;
2634 t->sw_if_index = sw_if_index0;
2635 t->next_index = next0;
2636 t->session_index = ~0;
2638 t->session_index = ses0 - dm0->sessions;
2641 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2643 /* verify speculative enqueue, maybe switch current next frame */
2644 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2645 to_next, n_left_to_next,
2649 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2652 vlib_node_increment_counter (vm, snat_det_in2out_node.index,
2653 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2655 return frame->n_vectors;
2658 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
2659 .function = snat_det_in2out_node_fn,
2660 .name = "snat-det-in2out",
2661 .vector_size = sizeof (u32),
2662 .format_trace = format_snat_in2out_trace,
2663 .type = VLIB_NODE_TYPE_INTERNAL,
2665 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2666 .error_strings = snat_in2out_error_strings,
2668 .runtime_data_bytes = sizeof (snat_runtime_t),
2672 /* edit / add dispositions here */
2674 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2675 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2676 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2680 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
2683 * Get address and port values to be used for packet SNAT translation
2684 * and create session if needed
2686 * @param[in,out] sm SNAT main
2687 * @param[in,out] node SNAT node runtime
2688 * @param[in] thread_index thread index
2689 * @param[in,out] b0 buffer containing packet to be translated
2690 * @param[out] p_proto protocol used for matching
2691 * @param[out] p_value address and port after NAT translation
2692 * @param[out] p_dont_translate if packet should not be translated
2693 * @param d optional parameter
2694 * @param e optional parameter
2696 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
2697 u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2698 snat_session_key_t *p_value,
2699 u8 *p_dont_translate, void *d, void *e)
2702 icmp46_header_t *icmp0;
2706 snat_det_out_key_t key0;
2707 u8 dont_translate = 0;
2709 icmp_echo_header_t *echo0, *inner_echo0 = 0;
2710 ip4_header_t *inner_ip0;
2711 void *l4_header = 0;
2712 icmp46_header_t *inner_icmp0;
2713 snat_det_map_t * dm0 = 0;
2714 ip4_address_t new_addr0;
2716 snat_det_session_t * ses0 = 0;
2717 ip4_address_t in_addr;
2720 ip0 = vlib_buffer_get_current (b0);
2721 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2722 echo0 = (icmp_echo_header_t *)(icmp0+1);
2723 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2724 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
2726 if (!icmp_is_error_message (icmp0))
2728 protocol = SNAT_PROTOCOL_ICMP;
2729 in_addr = ip0->src_address;
2730 in_port = echo0->identifier;
2734 inner_ip0 = (ip4_header_t *)(echo0+1);
2735 l4_header = ip4_next_header (inner_ip0);
2736 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2737 in_addr = inner_ip0->dst_address;
2740 case SNAT_PROTOCOL_ICMP:
2741 inner_icmp0 = (icmp46_header_t*)l4_header;
2742 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2743 in_port = inner_echo0->identifier;
2745 case SNAT_PROTOCOL_UDP:
2746 case SNAT_PROTOCOL_TCP:
2747 in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2750 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
2751 next0 = SNAT_IN2OUT_NEXT_DROP;
2756 dm0 = snat_det_map_by_user(sm, &in_addr);
2757 if (PREDICT_FALSE(!dm0))
2759 clib_warning("no match for internal host %U",
2760 format_ip4_address, &in_addr);
2761 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2762 IP_PROTOCOL_ICMP, rx_fib_index0)))
2767 next0 = SNAT_IN2OUT_NEXT_DROP;
2768 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2772 snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
2774 key0.ext_host_addr = ip0->dst_address;
2775 key0.ext_host_port = 0;
2777 ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
2778 if (PREDICT_FALSE(!ses0))
2780 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
2781 IP_PROTOCOL_ICMP, rx_fib_index0)))
2786 if (icmp0->type != ICMP4_echo_request)
2788 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2789 next0 = SNAT_IN2OUT_NEXT_DROP;
2792 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2794 key0.out_port = clib_host_to_net_u16 (lo_port0 +
2795 ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
2797 if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
2800 ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
2803 if (PREDICT_FALSE(!ses0))
2805 next0 = SNAT_IN2OUT_NEXT_DROP;
2806 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2811 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
2812 !icmp_is_error_message (icmp0)))
2814 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2815 next0 = SNAT_IN2OUT_NEXT_DROP;
2819 u32 now = (u32) vlib_time_now (sm->vlib_main);
2821 ses0->state = SNAT_SESSION_ICMP_ACTIVE;
2822 ses0->expire = now + sm->icmp_timeout;
2825 *p_proto = protocol;
2828 p_value->addr = new_addr0;
2829 p_value->fib_index = sm->outside_fib_index;
2830 p_value->port = ses0->out.out_port;
2832 *p_dont_translate = dont_translate;
2834 *(snat_det_session_t**)d = ses0;
2836 *(snat_det_map_t**)e = dm0;
2840 /**********************/
2841 /*** worker handoff ***/
2842 /**********************/
2844 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
2845 vlib_node_runtime_t * node,
2846 vlib_frame_t * frame,
2849 snat_main_t *sm = &snat_main;
2850 vlib_thread_main_t *tm = vlib_get_thread_main ();
2851 u32 n_left_from, *from, *to_next = 0;
2852 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2853 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2855 vlib_frame_queue_elt_t *hf = 0;
2856 vlib_frame_t *f = 0;
2858 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2859 u32 next_worker_index = 0;
2860 u32 current_worker_index = ~0;
2861 u32 thread_index = vlib_get_thread_index ();
2865 ASSERT (vec_len (sm->workers));
2869 fq_index = sm->fq_in2out_output_index;
2870 to_node_index = sm->in2out_output_node_index;
2874 fq_index = sm->fq_in2out_index;
2875 to_node_index = sm->in2out_node_index;
2878 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2880 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2882 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2883 sm->first_worker_index + sm->num_workers - 1,
2884 (vlib_frame_queue_t *) (~0));
2887 from = vlib_frame_vector_args (frame);
2888 n_left_from = frame->n_vectors;
2890 while (n_left_from > 0)
2903 b0 = vlib_get_buffer (vm, bi0);
2905 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2906 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2908 ip0 = vlib_buffer_get_current (b0);
2910 next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
2912 if (PREDICT_FALSE (next_worker_index != thread_index))
2916 if (next_worker_index != current_worker_index)
2919 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2921 hf = vlib_get_worker_handoff_queue_elt (fq_index,
2923 handoff_queue_elt_by_worker_index);
2925 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2926 to_next_worker = &hf->buffer_index[hf->n_vectors];
2927 current_worker_index = next_worker_index;
2930 /* enqueue to correct worker thread */
2931 to_next_worker[0] = bi0;
2933 n_left_to_next_worker--;
2935 if (n_left_to_next_worker == 0)
2937 hf->n_vectors = VLIB_FRAME_SIZE;
2938 vlib_put_frame_queue_elt (hf);
2939 current_worker_index = ~0;
2940 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2947 /* if this is 1st frame */
2950 f = vlib_get_frame_to_node (vm, to_node_index);
2951 to_next = vlib_frame_vector_args (f);
2959 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2960 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2962 snat_in2out_worker_handoff_trace_t *t =
2963 vlib_add_trace (vm, node, b0, sizeof (*t));
2964 t->next_worker_index = next_worker_index;
2965 t->do_handoff = do_handoff;
2970 vlib_put_frame_to_node (vm, to_node_index, f);
2973 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2975 /* Ship frames to the worker nodes */
2976 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2978 if (handoff_queue_elt_by_worker_index[i])
2980 hf = handoff_queue_elt_by_worker_index[i];
2982 * It works better to let the handoff node
2983 * rate-adapt, always ship the handoff queue element.
2985 if (1 || hf->n_vectors == hf->last_n_vectors)
2987 vlib_put_frame_queue_elt (hf);
2988 handoff_queue_elt_by_worker_index[i] = 0;
2991 hf->last_n_vectors = hf->n_vectors;
2993 congested_handoff_queue_by_worker_index[i] =
2994 (vlib_frame_queue_t *) (~0);
2997 current_worker_index = ~0;
2998 return frame->n_vectors;
3002 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3003 vlib_node_runtime_t * node,
3004 vlib_frame_t * frame)
3006 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3009 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3010 .function = snat_in2out_worker_handoff_fn,
3011 .name = "snat-in2out-worker-handoff",
3012 .vector_size = sizeof (u32),
3013 .format_trace = format_snat_in2out_worker_handoff_trace,
3014 .type = VLIB_NODE_TYPE_INTERNAL,
3023 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3024 snat_in2out_worker_handoff_fn);
3027 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3028 vlib_node_runtime_t * node,
3029 vlib_frame_t * frame)
3031 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3034 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3035 .function = snat_in2out_output_worker_handoff_fn,
3036 .name = "snat-in2out-output-worker-handoff",
3037 .vector_size = sizeof (u32),
3038 .format_trace = format_snat_in2out_worker_handoff_trace,
3039 .type = VLIB_NODE_TYPE_INTERNAL,
3048 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3049 snat_in2out_output_worker_handoff_fn);
3051 static_always_inline int
3052 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3054 snat_address_t * ap;
3055 clib_bihash_kv_8_8_t kv, value;
3056 snat_session_key_t m_key;
3058 vec_foreach (ap, sm->addresses)
3060 if (ap->addr.as_u32 == dst_addr->as_u32)
3064 m_key.addr.as_u32 = dst_addr->as_u32;
3065 m_key.fib_index = sm->outside_fib_index;
3068 kv.key = m_key.as_u64;
3069 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3076 snat_hairpin_dst_fn (vlib_main_t * vm,
3077 vlib_node_runtime_t * node,
3078 vlib_frame_t * frame)
3080 u32 n_left_from, * from, * to_next;
3081 snat_in2out_next_t next_index;
3082 u32 pkts_processed = 0;
3083 snat_main_t * sm = &snat_main;
3085 from = vlib_frame_vector_args (frame);
3086 n_left_from = frame->n_vectors;
3087 next_index = node->cached_next_index;
3089 while (n_left_from > 0)
3093 vlib_get_next_frame (vm, node, next_index,
3094 to_next, n_left_to_next);
3096 while (n_left_from > 0 && n_left_to_next > 0)
3104 /* speculatively enqueue b0 to the current next frame */
3110 n_left_to_next -= 1;
3112 b0 = vlib_get_buffer (vm, bi0);
3113 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3114 ip0 = vlib_buffer_get_current (b0);
3116 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3118 vnet_buffer (b0)->snat.flags = 0;
3119 if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3121 if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3123 udp_header_t * udp0 = ip4_next_header (ip0);
3124 tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3126 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3128 else if (proto0 == SNAT_PROTOCOL_ICMP)
3130 icmp46_header_t * icmp0 = ip4_next_header (ip0);
3132 snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3136 snat_hairpinning_unknown_proto (sm, b0, ip0);
3139 vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3140 clib_warning("is hairpinning");
3143 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3145 /* verify speculative enqueue, maybe switch current next frame */
3146 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3147 to_next, n_left_to_next,
3151 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3154 vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3155 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3157 return frame->n_vectors;
3160 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3161 .function = snat_hairpin_dst_fn,
3162 .name = "snat-hairpin-dst",
3163 .vector_size = sizeof (u32),
3164 .type = VLIB_NODE_TYPE_INTERNAL,
3165 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3166 .error_strings = snat_in2out_error_strings,
3169 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3170 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3174 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3175 snat_hairpin_dst_fn);
3178 snat_hairpin_src_fn (vlib_main_t * vm,
3179 vlib_node_runtime_t * node,
3180 vlib_frame_t * frame)
3182 u32 n_left_from, * from, * to_next;
3183 snat_in2out_next_t next_index;
3184 u32 pkts_processed = 0;
3185 snat_main_t *sm = &snat_main;
3187 from = vlib_frame_vector_args (frame);
3188 n_left_from = frame->n_vectors;
3189 next_index = node->cached_next_index;
3191 while (n_left_from > 0)
3195 vlib_get_next_frame (vm, node, next_index,
3196 to_next, n_left_to_next);
3198 while (n_left_from > 0 && n_left_to_next > 0)
3204 /* speculatively enqueue b0 to the current next frame */
3210 n_left_to_next -= 1;
3212 b0 = vlib_get_buffer (vm, bi0);
3213 next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3215 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) & SNAT_FLAG_HAIRPINNING))
3217 if (PREDICT_TRUE (sm->num_workers > 1))
3218 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3220 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3223 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3225 /* verify speculative enqueue, maybe switch current next frame */
3226 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3227 to_next, n_left_to_next,
3231 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3234 vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3235 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3237 return frame->n_vectors;
3240 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3241 .function = snat_hairpin_src_fn,
3242 .name = "snat-hairpin-src",
3243 .vector_size = sizeof (u32),
3244 .type = VLIB_NODE_TYPE_INTERNAL,
3245 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3246 .error_strings = snat_in2out_error_strings,
3247 .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3249 [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3250 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "snat-in2out-output",
3251 [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3252 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "snat-in2out-output-worker-handoff",
3256 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3257 snat_hairpin_src_fn);
3260 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3261 vlib_node_runtime_t * node,
3262 vlib_frame_t * frame)
3264 u32 n_left_from, * from, * to_next;
3265 snat_in2out_next_t next_index;
3266 u32 pkts_processed = 0;
3267 snat_main_t * sm = &snat_main;
3268 u32 stats_node_index;
3270 stats_node_index = snat_in2out_fast_node.index;
3272 from = vlib_frame_vector_args (frame);
3273 n_left_from = frame->n_vectors;
3274 next_index = node->cached_next_index;
3276 while (n_left_from > 0)
3280 vlib_get_next_frame (vm, node, next_index,
3281 to_next, n_left_to_next);
3283 while (n_left_from > 0 && n_left_to_next > 0)
3291 u32 new_addr0, old_addr0;
3292 u16 old_port0, new_port0;
3293 udp_header_t * udp0;
3294 tcp_header_t * tcp0;
3295 icmp46_header_t * icmp0;
3296 snat_session_key_t key0, sm0;
3300 /* speculatively enqueue b0 to the current next frame */
3306 n_left_to_next -= 1;
3308 b0 = vlib_get_buffer (vm, bi0);
3309 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3311 ip0 = vlib_buffer_get_current (b0);
3312 udp0 = ip4_next_header (ip0);
3313 tcp0 = (tcp_header_t *) udp0;
3314 icmp0 = (icmp46_header_t *) udp0;
3316 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3317 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3319 if (PREDICT_FALSE(ip0->ttl == 1))
3321 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3322 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3323 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3325 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3329 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3331 if (PREDICT_FALSE (proto0 == ~0))
3334 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3336 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3337 rx_fib_index0, node, next0, ~0, 0, 0);
3341 key0.addr = ip0->src_address;
3342 key0.port = udp0->src_port;
3343 key0.fib_index = rx_fib_index0;
3345 if (snat_static_mapping_match(sm, key0, &sm0, 0, 0))
3347 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3348 next0= SNAT_IN2OUT_NEXT_DROP;
3352 new_addr0 = sm0.addr.as_u32;
3353 new_port0 = sm0.port;
3354 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3355 old_addr0 = ip0->src_address.as_u32;
3356 ip0->src_address.as_u32 = new_addr0;
3358 sum0 = ip0->checksum;
3359 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3361 src_address /* changed member */);
3362 ip0->checksum = ip_csum_fold (sum0);
3364 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3366 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3368 old_port0 = tcp0->src_port;
3369 tcp0->src_port = new_port0;
3371 sum0 = tcp0->checksum;
3372 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3374 dst_address /* changed member */);
3375 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3376 ip4_header_t /* cheat */,
3377 length /* changed member */);
3378 tcp0->checksum = ip_csum_fold(sum0);
3382 old_port0 = udp0->src_port;
3383 udp0->src_port = new_port0;
3389 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3391 sum0 = tcp0->checksum;
3392 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3394 dst_address /* changed member */);
3395 tcp0->checksum = ip_csum_fold(sum0);
3400 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3403 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3404 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3406 snat_in2out_trace_t *t =
3407 vlib_add_trace (vm, node, b0, sizeof (*t));
3408 t->sw_if_index = sw_if_index0;
3409 t->next_index = next0;
3412 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3414 /* verify speculative enqueue, maybe switch current next frame */
3415 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3416 to_next, n_left_to_next,
3420 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3423 vlib_node_increment_counter (vm, stats_node_index,
3424 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3426 return frame->n_vectors;
3430 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
3431 .function = snat_in2out_fast_static_map_fn,
3432 .name = "snat-in2out-fast",
3433 .vector_size = sizeof (u32),
3434 .format_trace = format_snat_in2out_fast_trace,
3435 .type = VLIB_NODE_TYPE_INTERNAL,
3437 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3438 .error_strings = snat_in2out_error_strings,
3440 .runtime_data_bytes = sizeof (snat_runtime_t),
3442 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3444 /* edit / add dispositions here */
3446 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3447 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3448 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "snat-in2out-slowpath",
3449 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3453 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);