2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/fib/ip4_fib.h>
25 #include <nat/nat_ipfix_logging.h>
26 #include <nat/nat_det.h>
27 #include <nat/nat_reass.h>
29 #include <vppinfra/hash.h>
30 #include <vppinfra/error.h>
31 #include <vppinfra/elog.h>
38 } snat_in2out_trace_t;
41 u32 next_worker_index;
43 } snat_in2out_worker_handoff_trace_t;
45 /* packet trace format function */
46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
48 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
49 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
50 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
53 tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
55 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
56 t->sw_if_index, t->next_index, t->session_index);
61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
63 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
64 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
65 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
67 s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
68 t->sw_if_index, t->next_index);
73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
75 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
76 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
77 snat_in2out_worker_handoff_trace_t * t =
78 va_arg (*args, snat_in2out_worker_handoff_trace_t *);
81 m = t->do_handoff ? "next worker" : "same worker";
82 s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
91 } nat44_in2out_reass_trace_t;
93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
95 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
96 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
97 nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
99 s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
100 t->sw_if_index, t->next_index,
101 t->cached ? "cached" : "translated");
106 vlib_node_registration_t snat_in2out_node;
107 vlib_node_registration_t snat_in2out_slowpath_node;
108 vlib_node_registration_t snat_in2out_fast_node;
109 vlib_node_registration_t snat_in2out_worker_handoff_node;
110 vlib_node_registration_t snat_det_in2out_node;
111 vlib_node_registration_t snat_in2out_output_node;
112 vlib_node_registration_t snat_in2out_output_slowpath_node;
113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
114 vlib_node_registration_t snat_hairpin_dst_node;
115 vlib_node_registration_t snat_hairpin_src_node;
116 vlib_node_registration_t nat44_hairpinning_node;
117 vlib_node_registration_t nat44_in2out_reass_node;
120 #define foreach_snat_in2out_error \
121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
122 _(IN2OUT_PACKETS, "Good in2out packets processed") \
123 _(OUT_OF_PORTS, "Out of ports") \
124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \
125 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
126 _(NO_TRANSLATION, "No translation") \
127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
128 _(DROP_FRAGMENT, "Drop fragment") \
129 _(MAX_REASS, "Maximum reassemblies exceeded") \
130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
134 foreach_snat_in2out_error
137 } snat_in2out_error_t;
139 static char * snat_in2out_error_strings[] = {
140 #define _(sym,string) string,
141 foreach_snat_in2out_error
146 SNAT_IN2OUT_NEXT_LOOKUP,
147 SNAT_IN2OUT_NEXT_DROP,
148 SNAT_IN2OUT_NEXT_ICMP_ERROR,
149 SNAT_IN2OUT_NEXT_SLOW_PATH,
150 SNAT_IN2OUT_NEXT_REASS,
152 } snat_in2out_next_t;
155 SNAT_HAIRPIN_SRC_NEXT_DROP,
156 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
157 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
158 SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
159 SNAT_HAIRPIN_SRC_N_NEXT,
160 } snat_hairpin_next_t;
163 * @brief Check if packet should be translated
165 * Packets aimed at outside interface and external address with active session
166 * should be translated.
169 * @param rt NAT runtime data
170 * @param sw_if_index0 index of the inside interface
171 * @param ip0 IPv4 header
172 * @param proto0 NAT protocol
173 * @param rx_fib_index0 RX FIB index
175 * @returns 0 if packet should be translated otherwise 1
178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
179 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
185 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
187 .fp_proto = FIB_PROTOCOL_IP4,
190 .ip4.as_u32 = ip0->dst_address.as_u32,
194 /* Don't NAT packet aimed at the intfc address */
195 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
196 ip0->dst_address.as_u32)))
199 fei = fib_table_lookup (rx_fib_index0, &pfx);
200 if (FIB_NODE_INDEX_INVALID != fei)
202 u32 sw_if_index = fib_entry_get_resolving_interface (fei);
203 if (sw_if_index == ~0)
205 fei = fib_table_lookup (sm->outside_fib_index, &pfx);
206 if (FIB_NODE_INDEX_INVALID != fei)
207 sw_if_index = fib_entry_get_resolving_interface (fei);
210 pool_foreach (i, sm->interfaces,
212 /* NAT packet aimed at outside interface */
213 if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
223 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
224 u32 rx_fib_index0, u32 thread_index)
226 udp_header_t * udp0 = ip4_next_header (ip0);
227 snat_session_key_t key0, sm0;
228 clib_bihash_kv_8_8_t kv0, value0;
230 key0.addr = ip0->dst_address;
231 key0.port = udp0->dst_port;
232 key0.protocol = proto0;
233 key0.fib_index = sm->outside_fib_index;
234 kv0.key = key0.as_u64;
236 /* NAT packet aimed at external address if */
237 /* has active sessions */
238 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
241 /* or is static mappings */
242 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
248 if (sm->forwarding_enabled)
251 return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
255 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
258 snat_session_key_t * key0,
259 snat_session_t ** sessionp,
260 vlib_node_runtime_t * node,
266 clib_bihash_kv_8_8_t kv0;
267 snat_session_key_t key1;
268 u32 address_index = ~0;
269 u32 outside_fib_index;
271 udp_header_t * udp0 = ip4_next_header (ip0);
273 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
275 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
276 nat_ipfix_logging_max_sessions(sm->max_translations);
277 return SNAT_IN2OUT_NEXT_DROP;
280 p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
283 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
284 return SNAT_IN2OUT_NEXT_DROP;
286 outside_fib_index = p[0];
288 key1.protocol = key0->protocol;
290 u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
294 clib_warning ("create NAT user failed");
295 return SNAT_IN2OUT_NEXT_DROP;
298 /* First try to match static mapping by local address and port */
299 if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0))
301 /* Try to create dynamic translation */
302 if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
306 sm->per_thread_data[thread_index].snat_thread_index))
308 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
309 return SNAT_IN2OUT_NEXT_DROP;
315 u->nstaticsessions++;
318 s = nat_session_alloc_or_recycle (sm, u, thread_index);
321 clib_warning ("create NAT session failed");
322 return SNAT_IN2OUT_NEXT_DROP;
325 if (address_index == ~0)
326 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
327 s->outside_address_index = address_index;
330 s->out2in.protocol = key0->protocol;
331 s->out2in.fib_index = outside_fib_index;
332 s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
333 s->ext_host_port = udp0->dst_port;
336 /* Add to translation hashes */
337 kv0.key = s->in2out.as_u64;
338 kv0.value = s - sm->per_thread_data[thread_index].sessions;
339 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
341 clib_warning ("in2out key add failed");
343 kv0.key = s->out2in.as_u64;
344 kv0.value = s - sm->per_thread_data[thread_index].sessions;
346 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
348 clib_warning ("out2in key add failed");
351 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
352 s->out2in.addr.as_u32,
356 s->in2out.fib_index);
361 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
362 snat_session_key_t *p_key0)
364 icmp46_header_t *icmp0;
365 snat_session_key_t key0;
366 icmp_echo_header_t *echo0, *inner_echo0 = 0;
367 ip4_header_t *inner_ip0 = 0;
369 icmp46_header_t *inner_icmp0;
371 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
372 echo0 = (icmp_echo_header_t *)(icmp0+1);
374 if (!icmp_is_error_message (icmp0))
376 key0.protocol = SNAT_PROTOCOL_ICMP;
377 key0.addr = ip0->src_address;
378 key0.port = echo0->identifier;
382 inner_ip0 = (ip4_header_t *)(echo0+1);
383 l4_header = ip4_next_header (inner_ip0);
384 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
385 key0.addr = inner_ip0->dst_address;
386 switch (key0.protocol)
388 case SNAT_PROTOCOL_ICMP:
389 inner_icmp0 = (icmp46_header_t*)l4_header;
390 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
391 key0.port = inner_echo0->identifier;
393 case SNAT_PROTOCOL_UDP:
394 case SNAT_PROTOCOL_TCP:
395 key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
398 return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
402 return -1; /* success */
405 static_always_inline int
406 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
408 icmp46_header_t *icmp0;
409 nat_ed_ses_key_t key0;
410 icmp_echo_header_t *echo0, *inner_echo0 = 0;
411 ip4_header_t *inner_ip0 = 0;
413 icmp46_header_t *inner_icmp0;
415 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
416 echo0 = (icmp_echo_header_t *)(icmp0+1);
418 if (!icmp_is_error_message (icmp0))
420 key0.proto = IP_PROTOCOL_ICMP;
421 key0.l_addr = ip0->src_address;
422 key0.r_addr = ip0->dst_address;
423 key0.l_port = key0.r_port = echo0->identifier;
427 inner_ip0 = (ip4_header_t *)(echo0+1);
428 l4_header = ip4_next_header (inner_ip0);
429 key0.proto = inner_ip0->protocol;
430 key0.r_addr = inner_ip0->src_address;
431 key0.l_addr = inner_ip0->dst_address;
432 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
434 case SNAT_PROTOCOL_ICMP:
435 inner_icmp0 = (icmp46_header_t*)l4_header;
436 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
437 key0.r_port = key0.l_port = inner_echo0->identifier;
439 case SNAT_PROTOCOL_UDP:
440 case SNAT_PROTOCOL_TCP:
441 key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
442 key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
445 return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
453 * Get address and port values to be used for ICMP packet translation
454 * and create session if needed
456 * @param[in,out] sm NAT main
457 * @param[in,out] node NAT node runtime
458 * @param[in] thread_index thread index
459 * @param[in,out] b0 buffer containing packet to be translated
460 * @param[out] p_proto protocol used for matching
461 * @param[out] p_value address and port after NAT translation
462 * @param[out] p_dont_translate if packet should not be translated
463 * @param d optional parameter
464 * @param e optional parameter
466 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
467 u32 thread_index, vlib_buffer_t *b0,
468 ip4_header_t *ip0, u8 *p_proto,
469 snat_session_key_t *p_value,
470 u8 *p_dont_translate, void *d, void *e)
472 icmp46_header_t *icmp0;
475 snat_session_key_t key0;
476 snat_session_t *s0 = 0;
477 u8 dont_translate = 0;
478 clib_bihash_kv_8_8_t kv0, value0;
482 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
483 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
484 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
486 err = icmp_get_key (ip0, &key0);
489 b0->error = node->errors[err];
490 next0 = SNAT_IN2OUT_NEXT_DROP;
493 key0.fib_index = rx_fib_index0;
495 kv0.key = key0.as_u64;
497 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
500 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
501 IP_PROTOCOL_ICMP, rx_fib_index0, thread_index) &&
502 vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
508 if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
510 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
511 next0 = SNAT_IN2OUT_NEXT_DROP;
515 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
516 &s0, node, next0, thread_index);
518 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
523 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
524 icmp0->type != ICMP4_echo_reply &&
525 !icmp_is_error_message (icmp0)))
527 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
528 next0 = SNAT_IN2OUT_NEXT_DROP;
532 if (PREDICT_FALSE (value0.value == ~0ULL))
534 nat_ed_ses_key_t key;
535 clib_bihash_kv_16_8_t s_kv, s_value;
539 if (icmp_get_ed_key (ip0, &key))
541 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
542 next0 = SNAT_IN2OUT_NEXT_DROP;
545 key.fib_index = rx_fib_index0;
546 s_kv.key[0] = key.as_u64[0];
547 s_kv.key[1] = key.as_u64[1];
548 if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
549 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
553 next0 = SNAT_IN2OUT_NEXT_DROP;
558 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
563 *p_proto = key0.protocol;
565 *p_value = s0->out2in;
566 *p_dont_translate = dont_translate;
568 *(snat_session_t**)d = s0;
573 * Get address and port values to be used for ICMP packet translation
575 * @param[in] sm NAT main
576 * @param[in,out] node NAT node runtime
577 * @param[in] thread_index thread index
578 * @param[in,out] b0 buffer containing packet to be translated
579 * @param[out] p_proto protocol used for matching
580 * @param[out] p_value address and port after NAT translation
581 * @param[out] p_dont_translate if packet should not be translated
582 * @param d optional parameter
583 * @param e optional parameter
585 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
586 u32 thread_index, vlib_buffer_t *b0,
587 ip4_header_t *ip0, u8 *p_proto,
588 snat_session_key_t *p_value,
589 u8 *p_dont_translate, void *d, void *e)
591 icmp46_header_t *icmp0;
594 snat_session_key_t key0;
595 snat_session_key_t sm0;
596 u8 dont_translate = 0;
601 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
602 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
603 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
605 err = icmp_get_key (ip0, &key0);
608 b0->error = node->errors[err];
609 next0 = SNAT_IN2OUT_NEXT_DROP;
612 key0.fib_index = rx_fib_index0;
614 if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0))
616 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
617 IP_PROTOCOL_ICMP, rx_fib_index0)))
623 if (icmp_is_error_message (icmp0))
625 next0 = SNAT_IN2OUT_NEXT_DROP;
629 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
630 next0 = SNAT_IN2OUT_NEXT_DROP;
634 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
635 (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
636 !icmp_is_error_message (icmp0)))
638 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
639 next0 = SNAT_IN2OUT_NEXT_DROP;
646 *p_proto = key0.protocol;
647 *p_dont_translate = dont_translate;
651 static inline u32 icmp_in2out (snat_main_t *sm,
654 icmp46_header_t * icmp0,
657 vlib_node_runtime_t * node,
663 snat_session_key_t sm0;
665 icmp_echo_header_t *echo0, *inner_echo0 = 0;
666 ip4_header_t *inner_ip0;
668 icmp46_header_t *inner_icmp0;
670 u32 new_addr0, old_addr0;
671 u16 old_id0, new_id0;
676 echo0 = (icmp_echo_header_t *)(icmp0+1);
678 next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
679 &protocol, &sm0, &dont_translate, d, e);
682 if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
685 sum0 = ip_incremental_checksum (0, icmp0,
686 ntohs(ip0->length) - ip4_header_bytes (ip0));
687 checksum0 = ~ip_csum_fold (sum0);
688 if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
690 next0 = SNAT_IN2OUT_NEXT_DROP;
694 old_addr0 = ip0->src_address.as_u32;
695 new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
696 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
697 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
699 sum0 = ip0->checksum;
700 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
701 src_address /* changed member */);
702 ip0->checksum = ip_csum_fold (sum0);
704 if (!icmp_is_error_message (icmp0))
707 if (PREDICT_FALSE(new_id0 != echo0->identifier))
709 old_id0 = echo0->identifier;
711 echo0->identifier = new_id0;
713 sum0 = icmp0->checksum;
714 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
716 icmp0->checksum = ip_csum_fold (sum0);
721 inner_ip0 = (ip4_header_t *)(echo0+1);
722 l4_header = ip4_next_header (inner_ip0);
724 if (!ip4_header_checksum_is_valid (inner_ip0))
726 next0 = SNAT_IN2OUT_NEXT_DROP;
730 old_addr0 = inner_ip0->dst_address.as_u32;
731 inner_ip0->dst_address = sm0.addr;
732 new_addr0 = inner_ip0->dst_address.as_u32;
734 sum0 = icmp0->checksum;
735 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
736 dst_address /* changed member */);
737 icmp0->checksum = ip_csum_fold (sum0);
741 case SNAT_PROTOCOL_ICMP:
742 inner_icmp0 = (icmp46_header_t*)l4_header;
743 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
745 old_id0 = inner_echo0->identifier;
747 inner_echo0->identifier = new_id0;
749 sum0 = icmp0->checksum;
750 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
752 icmp0->checksum = ip_csum_fold (sum0);
754 case SNAT_PROTOCOL_UDP:
755 case SNAT_PROTOCOL_TCP:
756 old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
758 ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
760 sum0 = icmp0->checksum;
761 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
763 icmp0->checksum = ip_csum_fold (sum0);
777 * Hairpinning allows two endpoints on the internal side of the NAT to
778 * communicate even if they only use each other's external IP addresses
781 * @param sm NAT main.
782 * @param b0 Vlib buffer.
783 * @param ip0 IP header.
784 * @param udp0 UDP header.
785 * @param tcp0 TCP header.
786 * @param proto0 NAT protocol.
789 snat_hairpinning (snat_main_t *sm,
796 snat_session_key_t key0, sm0;
798 clib_bihash_kv_8_8_t kv0, value0;
800 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
801 u16 new_dst_port0, old_dst_port0;
803 key0.addr = ip0->dst_address;
804 key0.port = udp0->dst_port;
805 key0.protocol = proto0;
806 key0.fib_index = sm->outside_fib_index;
807 kv0.key = key0.as_u64;
809 /* Check if destination is static mappings */
810 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
812 new_dst_addr0 = sm0.addr.as_u32;
813 new_dst_port0 = sm0.port;
814 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
816 /* or active session */
819 if (sm->num_workers > 1)
820 ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
822 ti = sm->num_workers;
824 if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
828 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
829 new_dst_addr0 = s0->in2out.addr.as_u32;
830 new_dst_port0 = s0->in2out.port;
831 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
835 /* Destination is behind the same NAT, use internal address and port */
838 old_dst_addr0 = ip0->dst_address.as_u32;
839 ip0->dst_address.as_u32 = new_dst_addr0;
840 sum0 = ip0->checksum;
841 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
842 ip4_header_t, dst_address);
843 ip0->checksum = ip_csum_fold (sum0);
845 old_dst_port0 = tcp0->dst;
846 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
848 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
850 tcp0->dst = new_dst_port0;
851 sum0 = tcp0->checksum;
852 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
853 ip4_header_t, dst_address);
854 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
855 ip4_header_t /* cheat */, length);
856 tcp0->checksum = ip_csum_fold(sum0);
860 udp0->dst_port = new_dst_port0;
866 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
868 sum0 = tcp0->checksum;
869 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
870 ip4_header_t, dst_address);
871 tcp0->checksum = ip_csum_fold(sum0);
880 snat_icmp_hairpinning (snat_main_t *sm,
883 icmp46_header_t * icmp0)
885 snat_session_key_t key0, sm0;
886 clib_bihash_kv_8_8_t kv0, value0;
887 u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
891 if (!icmp_is_error_message (icmp0))
893 icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
894 u16 icmp_id0 = echo0->identifier;
895 key0.addr = ip0->dst_address;
896 key0.port = icmp_id0;
897 key0.protocol = SNAT_PROTOCOL_ICMP;
898 key0.fib_index = sm->outside_fib_index;
899 kv0.key = key0.as_u64;
901 if (sm->num_workers > 1)
902 ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
904 ti = sm->num_workers;
906 /* Check if destination is in active sessions */
907 if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
910 /* or static mappings */
911 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
913 new_dst_addr0 = sm0.addr.as_u32;
914 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
921 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
922 new_dst_addr0 = s0->in2out.addr.as_u32;
923 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
924 echo0->identifier = s0->in2out.port;
925 sum0 = icmp0->checksum;
926 sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
927 icmp_echo_header_t, identifier);
928 icmp0->checksum = ip_csum_fold (sum0);
931 /* Destination is behind the same NAT, use internal address and port */
934 old_dst_addr0 = ip0->dst_address.as_u32;
935 ip0->dst_address.as_u32 = new_dst_addr0;
936 sum0 = ip0->checksum;
937 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
938 ip4_header_t, dst_address);
939 ip0->checksum = ip_csum_fold (sum0);
945 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
948 icmp46_header_t * icmp0,
951 vlib_node_runtime_t * node,
955 snat_session_t ** p_s0)
957 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
958 next0, thread_index, p_s0, 0);
959 snat_session_t * s0 = *p_s0;
960 if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
963 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
964 snat_icmp_hairpinning(sm, b0, ip0, icmp0);
966 s0->last_heard = now;
968 s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
969 /* Per-user LRU list maintenance */
970 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
972 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
973 s0->per_user_list_head_index,
979 snat_hairpinning_unknown_proto (snat_main_t *sm,
983 u32 old_addr, new_addr = 0, ti = 0;
984 clib_bihash_kv_8_8_t kv, value;
985 clib_bihash_kv_16_8_t s_kv, s_value;
986 nat_ed_ses_key_t key;
987 snat_session_key_t m_key;
988 snat_static_mapping_t *m;
992 old_addr = ip->dst_address.as_u32;
993 key.l_addr.as_u32 = ip->dst_address.as_u32;
994 key.r_addr.as_u32 = ip->src_address.as_u32;
995 key.fib_index = sm->outside_fib_index;
996 key.proto = ip->protocol;
999 s_kv.key[0] = key.as_u64[0];
1000 s_kv.key[1] = key.as_u64[1];
1001 if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1003 m_key.addr = ip->dst_address;
1004 m_key.fib_index = sm->outside_fib_index;
1007 kv.key = m_key.as_u64;
1008 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1011 m = pool_elt_at_index (sm->static_mappings, value.value);
1012 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1013 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1014 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1018 if (sm->num_workers > 1)
1019 ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1021 ti = sm->num_workers;
1023 s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1024 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1025 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1026 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1029 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1030 ip->checksum = ip_csum_fold (sum);
1033 static snat_session_t *
1034 snat_in2out_unknown_proto (snat_main_t *sm,
1041 vlib_node_runtime_t * node)
1043 clib_bihash_kv_8_8_t kv, value;
1044 clib_bihash_kv_16_8_t s_kv, s_value;
1045 snat_static_mapping_t *m;
1046 snat_session_key_t m_key;
1047 u32 old_addr, new_addr = 0;
1050 dlist_elt_t *head, *elt;
1051 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1052 u32 elt_index, head_index, ses_index;
1054 nat_ed_ses_key_t key;
1055 u32 address_index = ~0;
1059 old_addr = ip->src_address.as_u32;
1061 key.l_addr = ip->src_address;
1062 key.r_addr = ip->dst_address;
1063 key.fib_index = rx_fib_index;
1064 key.proto = ip->protocol;
1067 s_kv.key[0] = key.as_u64[0];
1068 s_kv.key[1] = key.as_u64[1];
1070 if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1072 s = pool_elt_at_index (tsm->sessions, s_value.value);
1073 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1077 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1079 b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1080 nat_ipfix_logging_max_sessions(sm->max_translations);
1084 u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1088 clib_warning ("create NAT user failed");
1092 m_key.addr = ip->src_address;
1095 m_key.fib_index = rx_fib_index;
1096 kv.key = m_key.as_u64;
1098 /* Try to find static mapping first */
1099 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1101 m = pool_elt_at_index (sm->static_mappings, value.value);
1102 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1106 /* Fallback to 3-tuple key */
1109 /* Choose same out address as for TCP/UDP session to same destination */
1110 if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1112 head_index = u->sessions_per_user_list_head_index;
1113 head = pool_elt_at_index (tsm->list_pool, head_index);
1114 elt_index = head->next;
1115 elt = pool_elt_at_index (tsm->list_pool, elt_index);
1116 ses_index = elt->value;
1117 while (ses_index != ~0)
1119 s = pool_elt_at_index (tsm->sessions, ses_index);
1120 elt_index = elt->next;
1121 elt = pool_elt_at_index (tsm->list_pool, elt_index);
1122 ses_index = elt->value;
1124 if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1126 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1127 address_index = s->outside_address_index;
1129 key.fib_index = sm->outside_fib_index;
1130 key.l_addr.as_u32 = new_addr;
1131 s_kv.key[0] = key.as_u64[0];
1132 s_kv.key[1] = key.as_u64[1];
1133 if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1140 key.fib_index = sm->outside_fib_index;
1141 for (i = 0; i < vec_len (sm->addresses); i++)
1143 key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1144 s_kv.key[0] = key.as_u64[0];
1145 s_kv.key[1] = key.as_u64[1];
1146 if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1148 new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1157 s = nat_session_alloc_or_recycle (sm, u, thread_index);
1160 clib_warning ("create NAT session failed");
1164 s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1165 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1166 s->outside_address_index = address_index;
1167 s->out2in.addr.as_u32 = new_addr;
1168 s->out2in.fib_index = sm->outside_fib_index;
1169 s->in2out.addr.as_u32 = old_addr;
1170 s->in2out.fib_index = rx_fib_index;
1171 s->in2out.port = s->out2in.port = ip->protocol;
1174 u->nstaticsessions++;
1175 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1182 /* Add to lookup tables */
1183 key.l_addr.as_u32 = old_addr;
1184 key.r_addr = ip->dst_address;
1185 key.proto = ip->protocol;
1186 key.fib_index = rx_fib_index;
1187 s_kv.key[0] = key.as_u64[0];
1188 s_kv.key[1] = key.as_u64[1];
1189 s_kv.value = s - tsm->sessions;
1190 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1191 clib_warning ("in2out key add failed");
1193 key.l_addr.as_u32 = new_addr;
1194 key.fib_index = sm->outside_fib_index;
1195 s_kv.key[0] = key.as_u64[0];
1196 s_kv.key[1] = key.as_u64[1];
1197 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1198 clib_warning ("out2in key add failed");
1201 /* Update IP checksum */
1203 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1204 ip->checksum = ip_csum_fold (sum);
1207 s->last_heard = now;
1209 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1210 /* Per-user LRU list maintenance */
1211 clib_dlist_remove (tsm->list_pool, s->per_user_index);
1212 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1216 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1217 snat_hairpinning_unknown_proto(sm, b, ip);
1219 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1220 vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1225 static snat_session_t *
1226 snat_in2out_lb (snat_main_t *sm,
1233 vlib_node_runtime_t * node)
1235 nat_ed_ses_key_t key;
1236 clib_bihash_kv_16_8_t s_kv, s_value;
1237 udp_header_t *udp = ip4_next_header (ip);
1238 tcp_header_t *tcp = (tcp_header_t *) udp;
1239 snat_session_t *s = 0;
1240 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1241 u32 old_addr, new_addr;
1242 u16 new_port, old_port;
1244 u32 proto = ip_proto_to_snat_proto (ip->protocol);
1245 snat_session_key_t e_key, l_key;
1248 old_addr = ip->src_address.as_u32;
1250 key.l_addr = ip->src_address;
1251 key.r_addr = ip->dst_address;
1252 key.fib_index = rx_fib_index;
1253 key.proto = ip->protocol;
1254 key.r_port = udp->dst_port;
1255 key.l_port = udp->src_port;
1256 s_kv.key[0] = key.as_u64[0];
1257 s_kv.key[1] = key.as_u64[1];
1259 if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1261 s = pool_elt_at_index (tsm->sessions, s_value.value);
1265 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1267 b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1268 nat_ipfix_logging_max_sessions(sm->max_translations);
1272 l_key.addr = ip->src_address;
1273 l_key.port = udp->src_port;
1274 l_key.protocol = proto;
1275 l_key.fib_index = rx_fib_index;
1276 if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0))
1279 u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1283 clib_warning ("create NAT user failed");
1287 s = nat_session_alloc_or_recycle (sm, u, thread_index);
1290 clib_warning ("create NAT session failed");
1294 s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1295 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1296 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1297 s->outside_address_index = ~0;
1300 u->nstaticsessions++;
1302 /* Add to lookup tables */
1303 s_kv.value = s - tsm->sessions;
1304 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1305 clib_warning ("in2out-ed key add failed");
1307 key.l_addr = e_key.addr;
1308 key.fib_index = e_key.fib_index;
1309 key.l_port = e_key.port;
1310 s_kv.key[0] = key.as_u64[0];
1311 s_kv.key[1] = key.as_u64[1];
1312 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1313 clib_warning ("out2in-ed key add failed");
1316 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1318 /* Update IP checksum */
1320 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1321 if (is_twice_nat_session (s))
1322 sum = ip_csum_update (sum, ip->dst_address.as_u32,
1323 s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1324 ip->checksum = ip_csum_fold (sum);
1326 if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1328 old_port = tcp->src_port;
1329 tcp->src_port = s->out2in.port;
1330 new_port = tcp->src_port;
1332 sum = tcp->checksum;
1333 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1334 sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1335 if (is_twice_nat_session (s))
1337 sum = ip_csum_update (sum, ip->dst_address.as_u32,
1338 s->ext_host_addr.as_u32, ip4_header_t,
1340 sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1341 ip4_header_t, length);
1342 tcp->dst_port = s->ext_host_port;
1343 ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1345 tcp->checksum = ip_csum_fold(sum);
1349 udp->src_port = s->out2in.port;
1350 if (is_twice_nat_session (s))
1352 udp->dst_port = s->ext_host_port;
1353 ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1358 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1359 vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1362 s->last_heard = now;
1364 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1365 /* Per-user LRU list maintenance */
1366 clib_dlist_remove (tsm->list_pool, s->per_user_index);
1367 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1373 snat_in2out_node_fn_inline (vlib_main_t * vm,
1374 vlib_node_runtime_t * node,
1375 vlib_frame_t * frame, int is_slow_path,
1376 int is_output_feature)
1378 u32 n_left_from, * from, * to_next;
1379 snat_in2out_next_t next_index;
1380 u32 pkts_processed = 0;
1381 snat_main_t * sm = &snat_main;
1382 f64 now = vlib_time_now (vm);
1383 u32 stats_node_index;
1384 u32 thread_index = vlib_get_thread_index ();
1386 stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1387 snat_in2out_node.index;
1389 from = vlib_frame_vector_args (frame);
1390 n_left_from = frame->n_vectors;
1391 next_index = node->cached_next_index;
1393 while (n_left_from > 0)
1397 vlib_get_next_frame (vm, node, next_index,
1398 to_next, n_left_to_next);
1400 while (n_left_from >= 4 && n_left_to_next >= 2)
1403 vlib_buffer_t * b0, * b1;
1405 u32 sw_if_index0, sw_if_index1;
1406 ip4_header_t * ip0, * ip1;
1407 ip_csum_t sum0, sum1;
1408 u32 new_addr0, old_addr0, new_addr1, old_addr1;
1409 u16 old_port0, new_port0, old_port1, new_port1;
1410 udp_header_t * udp0, * udp1;
1411 tcp_header_t * tcp0, * tcp1;
1412 icmp46_header_t * icmp0, * icmp1;
1413 snat_session_key_t key0, key1;
1414 u32 rx_fib_index0, rx_fib_index1;
1416 snat_session_t * s0 = 0, * s1 = 0;
1417 clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1418 u32 iph_offset0 = 0, iph_offset1 = 0;
1420 /* Prefetch next iteration. */
1422 vlib_buffer_t * p2, * p3;
1424 p2 = vlib_get_buffer (vm, from[2]);
1425 p3 = vlib_get_buffer (vm, from[3]);
1427 vlib_prefetch_buffer_header (p2, LOAD);
1428 vlib_prefetch_buffer_header (p3, LOAD);
1430 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1431 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1434 /* speculatively enqueue b0 and b1 to the current next frame */
1435 to_next[0] = bi0 = from[0];
1436 to_next[1] = bi1 = from[1];
1440 n_left_to_next -= 2;
1442 b0 = vlib_get_buffer (vm, bi0);
1443 b1 = vlib_get_buffer (vm, bi1);
1445 if (is_output_feature)
1446 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1448 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1451 udp0 = ip4_next_header (ip0);
1452 tcp0 = (tcp_header_t *) udp0;
1453 icmp0 = (icmp46_header_t *) udp0;
1455 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1456 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1459 next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1461 if (PREDICT_FALSE(ip0->ttl == 1))
1463 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1464 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1465 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1467 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1471 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1473 /* Next configured feature, probably ip4-lookup */
1476 if (PREDICT_FALSE (proto0 == ~0))
1478 s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1479 thread_index, now, vm, node);
1481 next0 = SNAT_IN2OUT_NEXT_DROP;
1485 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1487 next0 = icmp_in2out_slow_path
1488 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1489 node, next0, now, thread_index, &s0);
1495 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1497 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1501 if (ip4_is_fragment (ip0))
1503 next0 = SNAT_IN2OUT_NEXT_REASS;
1508 key0.addr = ip0->src_address;
1509 key0.port = udp0->src_port;
1510 key0.protocol = proto0;
1511 key0.fib_index = rx_fib_index0;
1513 kv0.key = key0.as_u64;
1515 if (PREDICT_FALSE (clib_bihash_search_8_8 (
1516 &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1520 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1521 ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
1524 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1525 &s0, node, next0, thread_index);
1526 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1531 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1537 if (PREDICT_FALSE (value0.value == ~0ULL))
1541 s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1542 thread_index, now, vm, node);
1543 if (!s0 && !sm->forwarding_enabled)
1544 next0 = SNAT_IN2OUT_NEXT_DROP;
1549 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1555 s0 = pool_elt_at_index (
1556 sm->per_thread_data[thread_index].sessions,
1561 b0->flags |= VNET_BUFFER_F_IS_NATED;
1563 old_addr0 = ip0->src_address.as_u32;
1564 ip0->src_address = s0->out2in.addr;
1565 new_addr0 = ip0->src_address.as_u32;
1566 if (!is_output_feature)
1567 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1569 sum0 = ip0->checksum;
1570 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1572 src_address /* changed member */);
1573 ip0->checksum = ip_csum_fold (sum0);
1575 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1577 old_port0 = tcp0->src_port;
1578 tcp0->src_port = s0->out2in.port;
1579 new_port0 = tcp0->src_port;
1581 sum0 = tcp0->checksum;
1582 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1584 dst_address /* changed member */);
1585 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1586 ip4_header_t /* cheat */,
1587 length /* changed member */);
1588 tcp0->checksum = ip_csum_fold(sum0);
1592 old_port0 = udp0->src_port;
1593 udp0->src_port = s0->out2in.port;
1598 s0->last_heard = now;
1600 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1601 /* Per-user LRU list maintenance */
1602 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1603 s0->per_user_index);
1604 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1605 s0->per_user_list_head_index,
1606 s0->per_user_index);
1609 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1610 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1612 snat_in2out_trace_t *t =
1613 vlib_add_trace (vm, node, b0, sizeof (*t));
1614 t->is_slow_path = is_slow_path;
1615 t->sw_if_index = sw_if_index0;
1616 t->next_index = next0;
1617 t->session_index = ~0;
1619 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1622 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1624 if (is_output_feature)
1625 iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1627 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1630 udp1 = ip4_next_header (ip1);
1631 tcp1 = (tcp_header_t *) udp1;
1632 icmp1 = (icmp46_header_t *) udp1;
1634 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1635 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1638 if (PREDICT_FALSE(ip1->ttl == 1))
1640 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1641 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1642 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1644 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1648 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1650 /* Next configured feature, probably ip4-lookup */
1653 if (PREDICT_FALSE (proto1 == ~0))
1655 s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1656 thread_index, now, vm, node);
1658 next1 = SNAT_IN2OUT_NEXT_DROP;
1662 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1664 next1 = icmp_in2out_slow_path
1665 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1666 next1, now, thread_index, &s1);
1672 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1674 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1678 if (ip4_is_fragment (ip1))
1680 next1 = SNAT_IN2OUT_NEXT_REASS;
1685 b1->flags |= VNET_BUFFER_F_IS_NATED;
1687 key1.addr = ip1->src_address;
1688 key1.port = udp1->src_port;
1689 key1.protocol = proto1;
1690 key1.fib_index = rx_fib_index1;
1692 kv1.key = key1.as_u64;
1694 if (PREDICT_FALSE(clib_bihash_search_8_8 (
1695 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1699 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1700 ip1, proto1, rx_fib_index1, thread_index)) && !is_output_feature)
1703 next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1704 &s1, node, next1, thread_index);
1705 if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1710 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1716 if (PREDICT_FALSE (value1.value == ~0ULL))
1720 s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1721 thread_index, now, vm, node);
1722 if (!s1 && !sm->forwarding_enabled)
1723 next1 = SNAT_IN2OUT_NEXT_DROP;
1728 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1734 s1 = pool_elt_at_index (
1735 sm->per_thread_data[thread_index].sessions,
1740 old_addr1 = ip1->src_address.as_u32;
1741 ip1->src_address = s1->out2in.addr;
1742 new_addr1 = ip1->src_address.as_u32;
1743 if (!is_output_feature)
1744 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1746 sum1 = ip1->checksum;
1747 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1749 src_address /* changed member */);
1750 ip1->checksum = ip_csum_fold (sum1);
1752 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1754 old_port1 = tcp1->src_port;
1755 tcp1->src_port = s1->out2in.port;
1756 new_port1 = tcp1->src_port;
1758 sum1 = tcp1->checksum;
1759 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1761 dst_address /* changed member */);
1762 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1763 ip4_header_t /* cheat */,
1764 length /* changed member */);
1765 tcp1->checksum = ip_csum_fold(sum1);
1769 old_port1 = udp1->src_port;
1770 udp1->src_port = s1->out2in.port;
1775 s1->last_heard = now;
1777 s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1778 /* Per-user LRU list maintenance */
1779 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1780 s1->per_user_index);
1781 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1782 s1->per_user_list_head_index,
1783 s1->per_user_index);
1786 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1787 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1789 snat_in2out_trace_t *t =
1790 vlib_add_trace (vm, node, b1, sizeof (*t));
1791 t->sw_if_index = sw_if_index1;
1792 t->next_index = next1;
1793 t->session_index = ~0;
1795 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1798 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1800 /* verify speculative enqueues, maybe switch current next frame */
1801 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1802 to_next, n_left_to_next,
1803 bi0, bi1, next0, next1);
1806 while (n_left_from > 0 && n_left_to_next > 0)
1814 u32 new_addr0, old_addr0;
1815 u16 old_port0, new_port0;
1816 udp_header_t * udp0;
1817 tcp_header_t * tcp0;
1818 icmp46_header_t * icmp0;
1819 snat_session_key_t key0;
1822 snat_session_t * s0 = 0;
1823 clib_bihash_kv_8_8_t kv0, value0;
1824 u32 iph_offset0 = 0;
1826 /* speculatively enqueue b0 to the current next frame */
1832 n_left_to_next -= 1;
1834 b0 = vlib_get_buffer (vm, bi0);
1835 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1837 if (is_output_feature)
1838 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1840 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1843 udp0 = ip4_next_header (ip0);
1844 tcp0 = (tcp_header_t *) udp0;
1845 icmp0 = (icmp46_header_t *) udp0;
1847 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1848 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1851 if (PREDICT_FALSE(ip0->ttl == 1))
1853 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1854 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1855 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1857 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1861 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1863 /* Next configured feature, probably ip4-lookup */
1866 if (PREDICT_FALSE (proto0 == ~0))
1868 s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1869 thread_index, now, vm, node);
1871 next0 = SNAT_IN2OUT_NEXT_DROP;
1875 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1877 next0 = icmp_in2out_slow_path
1878 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1879 next0, now, thread_index, &s0);
1885 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1887 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1891 if (ip4_is_fragment (ip0))
1893 next0 = SNAT_IN2OUT_NEXT_REASS;
1898 key0.addr = ip0->src_address;
1899 key0.port = udp0->src_port;
1900 key0.protocol = proto0;
1901 key0.fib_index = rx_fib_index0;
1903 kv0.key = key0.as_u64;
1905 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1910 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1911 ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
1914 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1915 &s0, node, next0, thread_index);
1917 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1922 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1928 if (PREDICT_FALSE (value0.value == ~0ULL))
1932 s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1933 thread_index, now, vm, node);
1934 if (!s0 && !sm->forwarding_enabled)
1935 next0 = SNAT_IN2OUT_NEXT_DROP;
1940 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1946 s0 = pool_elt_at_index (
1947 sm->per_thread_data[thread_index].sessions,
1952 b0->flags |= VNET_BUFFER_F_IS_NATED;
1954 old_addr0 = ip0->src_address.as_u32;
1955 ip0->src_address = s0->out2in.addr;
1956 new_addr0 = ip0->src_address.as_u32;
1957 if (!is_output_feature)
1958 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1960 sum0 = ip0->checksum;
1961 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1963 src_address /* changed member */);
1964 ip0->checksum = ip_csum_fold (sum0);
1966 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1968 old_port0 = tcp0->src_port;
1969 tcp0->src_port = s0->out2in.port;
1970 new_port0 = tcp0->src_port;
1972 sum0 = tcp0->checksum;
1973 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1975 dst_address /* changed member */);
1976 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1977 ip4_header_t /* cheat */,
1978 length /* changed member */);
1979 tcp0->checksum = ip_csum_fold(sum0);
1983 old_port0 = udp0->src_port;
1984 udp0->src_port = s0->out2in.port;
1989 s0->last_heard = now;
1991 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1992 /* Per-user LRU list maintenance */
1993 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1994 s0->per_user_index);
1995 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1996 s0->per_user_list_head_index,
1997 s0->per_user_index);
2000 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2001 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2003 snat_in2out_trace_t *t =
2004 vlib_add_trace (vm, node, b0, sizeof (*t));
2005 t->is_slow_path = is_slow_path;
2006 t->sw_if_index = sw_if_index0;
2007 t->next_index = next0;
2008 t->session_index = ~0;
2010 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2013 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2015 /* verify speculative enqueue, maybe switch current next frame */
2016 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2017 to_next, n_left_to_next,
2021 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2024 vlib_node_increment_counter (vm, stats_node_index,
2025 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2027 return frame->n_vectors;
2031 snat_in2out_fast_path_fn (vlib_main_t * vm,
2032 vlib_node_runtime_t * node,
2033 vlib_frame_t * frame)
2035 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2038 VLIB_REGISTER_NODE (snat_in2out_node) = {
2039 .function = snat_in2out_fast_path_fn,
2040 .name = "nat44-in2out",
2041 .vector_size = sizeof (u32),
2042 .format_trace = format_snat_in2out_trace,
2043 .type = VLIB_NODE_TYPE_INTERNAL,
2045 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2046 .error_strings = snat_in2out_error_strings,
2048 .runtime_data_bytes = sizeof (snat_runtime_t),
2050 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2052 /* edit / add dispositions here */
2054 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2055 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2056 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2057 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2058 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2062 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2065 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2066 vlib_node_runtime_t * node,
2067 vlib_frame_t * frame)
2069 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2072 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2073 .function = snat_in2out_output_fast_path_fn,
2074 .name = "nat44-in2out-output",
2075 .vector_size = sizeof (u32),
2076 .format_trace = format_snat_in2out_trace,
2077 .type = VLIB_NODE_TYPE_INTERNAL,
2079 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2080 .error_strings = snat_in2out_error_strings,
2082 .runtime_data_bytes = sizeof (snat_runtime_t),
2084 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2086 /* edit / add dispositions here */
2088 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2089 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2090 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2091 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2092 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2096 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2097 snat_in2out_output_fast_path_fn);
2100 snat_in2out_slow_path_fn (vlib_main_t * vm,
2101 vlib_node_runtime_t * node,
2102 vlib_frame_t * frame)
2104 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2107 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2108 .function = snat_in2out_slow_path_fn,
2109 .name = "nat44-in2out-slowpath",
2110 .vector_size = sizeof (u32),
2111 .format_trace = format_snat_in2out_trace,
2112 .type = VLIB_NODE_TYPE_INTERNAL,
2114 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2115 .error_strings = snat_in2out_error_strings,
2117 .runtime_data_bytes = sizeof (snat_runtime_t),
2119 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2121 /* edit / add dispositions here */
2123 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2124 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2125 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2126 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2127 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2131 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2132 snat_in2out_slow_path_fn);
2135 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2136 vlib_node_runtime_t * node,
2137 vlib_frame_t * frame)
2139 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2142 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2143 .function = snat_in2out_output_slow_path_fn,
2144 .name = "nat44-in2out-output-slowpath",
2145 .vector_size = sizeof (u32),
2146 .format_trace = format_snat_in2out_trace,
2147 .type = VLIB_NODE_TYPE_INTERNAL,
2149 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2150 .error_strings = snat_in2out_error_strings,
2152 .runtime_data_bytes = sizeof (snat_runtime_t),
2154 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2156 /* edit / add dispositions here */
2158 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2159 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2160 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2161 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2162 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2166 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2167 snat_in2out_output_slow_path_fn);
2169 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2172 nat44_hairpinning_fn (vlib_main_t * vm,
2173 vlib_node_runtime_t * node,
2174 vlib_frame_t * frame)
2176 u32 n_left_from, * from, * to_next;
2177 snat_in2out_next_t next_index;
2178 u32 pkts_processed = 0;
2179 snat_main_t * sm = &snat_main;
2180 vnet_feature_main_t *fm = &feature_main;
2181 u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2182 vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2184 from = vlib_frame_vector_args (frame);
2185 n_left_from = frame->n_vectors;
2186 next_index = node->cached_next_index;
2188 while (n_left_from > 0)
2192 vlib_get_next_frame (vm, node, next_index,
2193 to_next, n_left_to_next);
2195 while (n_left_from > 0 && n_left_to_next > 0)
2202 udp_header_t * udp0;
2203 tcp_header_t * tcp0;
2205 /* speculatively enqueue b0 to the current next frame */
2211 n_left_to_next -= 1;
2213 b0 = vlib_get_buffer (vm, bi0);
2214 ip0 = vlib_buffer_get_current (b0);
2215 udp0 = ip4_next_header (ip0);
2216 tcp0 = (tcp_header_t *) udp0;
2218 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2220 vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2223 if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2224 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2226 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2228 /* verify speculative enqueue, maybe switch current next frame */
2229 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2230 to_next, n_left_to_next,
2234 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2237 vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2238 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2240 return frame->n_vectors;
2243 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2244 .function = nat44_hairpinning_fn,
2245 .name = "nat44-hairpinning",
2246 .vector_size = sizeof (u32),
2247 .type = VLIB_NODE_TYPE_INTERNAL,
2248 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2249 .error_strings = snat_in2out_error_strings,
2252 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2253 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2257 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2258 nat44_hairpinning_fn);
2261 nat44_reass_hairpinning (snat_main_t *sm,
2268 snat_session_key_t key0, sm0;
2269 snat_session_t * s0;
2270 clib_bihash_kv_8_8_t kv0, value0;
2272 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2273 u16 new_dst_port0, old_dst_port0;
2274 udp_header_t * udp0;
2275 tcp_header_t * tcp0;
2277 key0.addr = ip0->dst_address;
2279 key0.protocol = proto0;
2280 key0.fib_index = sm->outside_fib_index;
2281 kv0.key = key0.as_u64;
2283 udp0 = ip4_next_header (ip0);
2285 /* Check if destination is static mappings */
2286 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
2288 new_dst_addr0 = sm0.addr.as_u32;
2289 new_dst_port0 = sm0.port;
2290 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2292 /* or active sessions */
2295 if (sm->num_workers > 1)
2296 ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2298 ti = sm->num_workers;
2300 if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2303 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2304 new_dst_addr0 = s0->in2out.addr.as_u32;
2305 new_dst_port0 = s0->in2out.port;
2306 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2310 /* Destination is behind the same NAT, use internal address and port */
2313 old_dst_addr0 = ip0->dst_address.as_u32;
2314 ip0->dst_address.as_u32 = new_dst_addr0;
2315 sum0 = ip0->checksum;
2316 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2317 ip4_header_t, dst_address);
2318 ip0->checksum = ip_csum_fold (sum0);
2320 old_dst_port0 = dport;
2321 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2322 ip4_is_first_fragment (ip0)))
2324 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2326 tcp0 = ip4_next_header (ip0);
2327 tcp0->dst = new_dst_port0;
2328 sum0 = tcp0->checksum;
2329 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2330 ip4_header_t, dst_address);
2331 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2332 ip4_header_t /* cheat */, length);
2333 tcp0->checksum = ip_csum_fold(sum0);
2337 udp0->dst_port = new_dst_port0;
2343 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2345 tcp0 = ip4_next_header (ip0);
2346 sum0 = tcp0->checksum;
2347 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2348 ip4_header_t, dst_address);
2349 tcp0->checksum = ip_csum_fold(sum0);
2356 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2357 vlib_node_runtime_t * node,
2358 vlib_frame_t * frame)
2360 u32 n_left_from, *from, *to_next;
2361 snat_in2out_next_t next_index;
2362 u32 pkts_processed = 0;
2363 snat_main_t *sm = &snat_main;
2364 f64 now = vlib_time_now (vm);
2365 u32 thread_index = vlib_get_thread_index ();
2366 snat_main_per_thread_data_t *per_thread_data =
2367 &sm->per_thread_data[thread_index];
2368 u32 *fragments_to_drop = 0;
2369 u32 *fragments_to_loopback = 0;
2371 from = vlib_frame_vector_args (frame);
2372 n_left_from = frame->n_vectors;
2373 next_index = node->cached_next_index;
2375 while (n_left_from > 0)
2379 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2381 while (n_left_from > 0 && n_left_to_next > 0)
2383 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2388 nat_reass_ip4_t *reass0;
2389 udp_header_t * udp0;
2390 tcp_header_t * tcp0;
2391 snat_session_key_t key0;
2392 clib_bihash_kv_8_8_t kv0, value0;
2393 snat_session_t * s0 = 0;
2394 u16 old_port0, new_port0;
2397 /* speculatively enqueue b0 to the current next frame */
2403 n_left_to_next -= 1;
2405 b0 = vlib_get_buffer (vm, bi0);
2406 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2408 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2409 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2412 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2414 next0 = SNAT_IN2OUT_NEXT_DROP;
2415 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2419 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2420 udp0 = ip4_next_header (ip0);
2421 tcp0 = (tcp_header_t *) udp0;
2422 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2424 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2429 &fragments_to_drop);
2431 if (PREDICT_FALSE (!reass0))
2433 next0 = SNAT_IN2OUT_NEXT_DROP;
2434 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2438 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2440 key0.addr = ip0->src_address;
2441 key0.port = udp0->src_port;
2442 key0.protocol = proto0;
2443 key0.fib_index = rx_fib_index0;
2444 kv0.key = key0.as_u64;
2446 if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2448 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2449 ip0, proto0, rx_fib_index0, thread_index)))
2452 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2453 &s0, node, next0, thread_index);
2455 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2458 reass0->sess_index = s0 - per_thread_data->sessions;
2462 s0 = pool_elt_at_index (per_thread_data->sessions,
2464 reass0->sess_index = value0.value;
2466 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2470 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2472 if (nat_ip4_reass_add_fragment (reass0, bi0))
2474 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2475 next0 = SNAT_IN2OUT_NEXT_DROP;
2481 s0 = pool_elt_at_index (per_thread_data->sessions,
2482 reass0->sess_index);
2485 old_addr0 = ip0->src_address.as_u32;
2486 ip0->src_address = s0->out2in.addr;
2487 new_addr0 = ip0->src_address.as_u32;
2488 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2490 sum0 = ip0->checksum;
2491 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2493 src_address /* changed member */);
2494 ip0->checksum = ip_csum_fold (sum0);
2496 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2498 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2500 old_port0 = tcp0->src_port;
2501 tcp0->src_port = s0->out2in.port;
2502 new_port0 = tcp0->src_port;
2504 sum0 = tcp0->checksum;
2505 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2507 dst_address /* changed member */);
2508 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2509 ip4_header_t /* cheat */,
2510 length /* changed member */);
2511 tcp0->checksum = ip_csum_fold(sum0);
2515 old_port0 = udp0->src_port;
2516 udp0->src_port = s0->out2in.port;
2522 nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2523 s0->ext_host_port, proto0);
2526 s0->last_heard = now;
2528 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2529 /* Per-user LRU list maintenance */
2530 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2531 s0->per_user_index);
2532 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2533 s0->per_user_list_head_index,
2534 s0->per_user_index);
2537 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2538 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2540 nat44_in2out_reass_trace_t *t =
2541 vlib_add_trace (vm, node, b0, sizeof (*t));
2542 t->cached = cached0;
2543 t->sw_if_index = sw_if_index0;
2544 t->next_index = next0;
2554 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2556 /* verify speculative enqueue, maybe switch current next frame */
2557 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2558 to_next, n_left_to_next,
2562 if (n_left_from == 0 && vec_len (fragments_to_loopback))
2564 from = vlib_frame_vector_args (frame);
2565 u32 len = vec_len (fragments_to_loopback);
2566 if (len <= VLIB_FRAME_SIZE)
2568 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2570 vec_reset_length (fragments_to_loopback);
2575 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2576 sizeof (u32) * VLIB_FRAME_SIZE);
2577 n_left_from = VLIB_FRAME_SIZE;
2578 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2583 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2586 vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2587 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2590 nat_send_all_to_node (vm, fragments_to_drop, node,
2591 &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2592 SNAT_IN2OUT_NEXT_DROP);
2594 vec_free (fragments_to_drop);
2595 vec_free (fragments_to_loopback);
2596 return frame->n_vectors;
2599 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2600 .function = nat44_in2out_reass_node_fn,
2601 .name = "nat44-in2out-reass",
2602 .vector_size = sizeof (u32),
2603 .format_trace = format_nat44_in2out_reass_trace,
2604 .type = VLIB_NODE_TYPE_INTERNAL,
2606 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2607 .error_strings = snat_in2out_error_strings,
2609 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2611 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2612 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2613 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2614 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2615 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2619 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2620 nat44_in2out_reass_node_fn);
2622 /**************************/
2623 /*** deterministic mode ***/
2624 /**************************/
2626 snat_det_in2out_node_fn (vlib_main_t * vm,
2627 vlib_node_runtime_t * node,
2628 vlib_frame_t * frame)
2630 u32 n_left_from, * from, * to_next;
2631 snat_in2out_next_t next_index;
2632 u32 pkts_processed = 0;
2633 snat_main_t * sm = &snat_main;
2634 u32 now = (u32) vlib_time_now (vm);
2635 u32 thread_index = vlib_get_thread_index ();
2637 from = vlib_frame_vector_args (frame);
2638 n_left_from = frame->n_vectors;
2639 next_index = node->cached_next_index;
2641 while (n_left_from > 0)
2645 vlib_get_next_frame (vm, node, next_index,
2646 to_next, n_left_to_next);
2648 while (n_left_from >= 4 && n_left_to_next >= 2)
2651 vlib_buffer_t * b0, * b1;
2653 u32 sw_if_index0, sw_if_index1;
2654 ip4_header_t * ip0, * ip1;
2655 ip_csum_t sum0, sum1;
2656 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2657 u16 old_port0, new_port0, lo_port0, i0;
2658 u16 old_port1, new_port1, lo_port1, i1;
2659 udp_header_t * udp0, * udp1;
2660 tcp_header_t * tcp0, * tcp1;
2662 snat_det_out_key_t key0, key1;
2663 snat_det_map_t * dm0, * dm1;
2664 snat_det_session_t * ses0 = 0, * ses1 = 0;
2665 u32 rx_fib_index0, rx_fib_index1;
2666 icmp46_header_t * icmp0, * icmp1;
2668 /* Prefetch next iteration. */
2670 vlib_buffer_t * p2, * p3;
2672 p2 = vlib_get_buffer (vm, from[2]);
2673 p3 = vlib_get_buffer (vm, from[3]);
2675 vlib_prefetch_buffer_header (p2, LOAD);
2676 vlib_prefetch_buffer_header (p3, LOAD);
2678 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2679 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2682 /* speculatively enqueue b0 and b1 to the current next frame */
2683 to_next[0] = bi0 = from[0];
2684 to_next[1] = bi1 = from[1];
2688 n_left_to_next -= 2;
2690 b0 = vlib_get_buffer (vm, bi0);
2691 b1 = vlib_get_buffer (vm, bi1);
2693 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2694 next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2696 ip0 = vlib_buffer_get_current (b0);
2697 udp0 = ip4_next_header (ip0);
2698 tcp0 = (tcp_header_t *) udp0;
2700 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2702 if (PREDICT_FALSE(ip0->ttl == 1))
2704 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2705 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2706 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2708 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2712 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2714 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2716 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2717 icmp0 = (icmp46_header_t *) udp0;
2719 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2720 rx_fib_index0, node, next0, thread_index,
2725 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2726 if (PREDICT_FALSE(!dm0))
2728 clib_warning("no match for internal host %U",
2729 format_ip4_address, &ip0->src_address);
2730 next0 = SNAT_IN2OUT_NEXT_DROP;
2731 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2735 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2737 key0.ext_host_addr = ip0->dst_address;
2738 key0.ext_host_port = tcp0->dst;
2740 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2741 if (PREDICT_FALSE(!ses0))
2743 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2745 key0.out_port = clib_host_to_net_u16 (lo_port0 +
2746 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2748 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2751 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2754 if (PREDICT_FALSE(!ses0))
2756 /* too many sessions for user, send ICMP error packet */
2758 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2759 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2760 ICMP4_destination_unreachable_destination_unreachable_host,
2762 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2767 new_port0 = ses0->out.out_port;
2769 old_addr0.as_u32 = ip0->src_address.as_u32;
2770 ip0->src_address.as_u32 = new_addr0.as_u32;
2771 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2773 sum0 = ip0->checksum;
2774 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2776 src_address /* changed member */);
2777 ip0->checksum = ip_csum_fold (sum0);
2779 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2781 if (tcp0->flags & TCP_FLAG_SYN)
2782 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2783 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2784 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2785 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2786 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2787 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2788 snat_det_ses_close(dm0, ses0);
2789 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2790 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2791 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2792 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2794 old_port0 = tcp0->src;
2795 tcp0->src = new_port0;
2797 sum0 = tcp0->checksum;
2798 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2800 dst_address /* changed member */);
2801 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2802 ip4_header_t /* cheat */,
2803 length /* changed member */);
2804 tcp0->checksum = ip_csum_fold(sum0);
2808 ses0->state = SNAT_SESSION_UDP_ACTIVE;
2809 old_port0 = udp0->src_port;
2810 udp0->src_port = new_port0;
2816 case SNAT_SESSION_UDP_ACTIVE:
2817 ses0->expire = now + sm->udp_timeout;
2819 case SNAT_SESSION_TCP_SYN_SENT:
2820 case SNAT_SESSION_TCP_FIN_WAIT:
2821 case SNAT_SESSION_TCP_CLOSE_WAIT:
2822 case SNAT_SESSION_TCP_LAST_ACK:
2823 ses0->expire = now + sm->tcp_transitory_timeout;
2825 case SNAT_SESSION_TCP_ESTABLISHED:
2826 ses0->expire = now + sm->tcp_established_timeout;
2831 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2832 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2834 snat_in2out_trace_t *t =
2835 vlib_add_trace (vm, node, b0, sizeof (*t));
2836 t->is_slow_path = 0;
2837 t->sw_if_index = sw_if_index0;
2838 t->next_index = next0;
2839 t->session_index = ~0;
2841 t->session_index = ses0 - dm0->sessions;
2844 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2846 ip1 = vlib_buffer_get_current (b1);
2847 udp1 = ip4_next_header (ip1);
2848 tcp1 = (tcp_header_t *) udp1;
2850 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2852 if (PREDICT_FALSE(ip1->ttl == 1))
2854 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2855 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2856 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2858 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2862 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2864 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2866 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2867 icmp1 = (icmp46_header_t *) udp1;
2869 next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2870 rx_fib_index1, node, next1, thread_index,
2875 dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2876 if (PREDICT_FALSE(!dm1))
2878 clib_warning("no match for internal host %U",
2879 format_ip4_address, &ip0->src_address);
2880 next1 = SNAT_IN2OUT_NEXT_DROP;
2881 b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2885 snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
2887 key1.ext_host_addr = ip1->dst_address;
2888 key1.ext_host_port = tcp1->dst;
2890 ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
2891 if (PREDICT_FALSE(!ses1))
2893 for (i1 = 0; i1 < dm1->ports_per_host; i1++)
2895 key1.out_port = clib_host_to_net_u16 (lo_port1 +
2896 ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
2898 if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
2901 ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
2904 if (PREDICT_FALSE(!ses1))
2906 /* too many sessions for user, send ICMP error packet */
2908 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2909 icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
2910 ICMP4_destination_unreachable_destination_unreachable_host,
2912 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2917 new_port1 = ses1->out.out_port;
2919 old_addr1.as_u32 = ip1->src_address.as_u32;
2920 ip1->src_address.as_u32 = new_addr1.as_u32;
2921 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2923 sum1 = ip1->checksum;
2924 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2926 src_address /* changed member */);
2927 ip1->checksum = ip_csum_fold (sum1);
2929 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2931 if (tcp1->flags & TCP_FLAG_SYN)
2932 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
2933 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
2934 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2935 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2936 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
2937 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
2938 snat_det_ses_close(dm1, ses1);
2939 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2940 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
2941 else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
2942 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2944 old_port1 = tcp1->src;
2945 tcp1->src = new_port1;
2947 sum1 = tcp1->checksum;
2948 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2950 dst_address /* changed member */);
2951 sum1 = ip_csum_update (sum1, old_port1, new_port1,
2952 ip4_header_t /* cheat */,
2953 length /* changed member */);
2954 tcp1->checksum = ip_csum_fold(sum1);
2958 ses1->state = SNAT_SESSION_UDP_ACTIVE;
2959 old_port1 = udp1->src_port;
2960 udp1->src_port = new_port1;
2966 case SNAT_SESSION_UDP_ACTIVE:
2967 ses1->expire = now + sm->udp_timeout;
2969 case SNAT_SESSION_TCP_SYN_SENT:
2970 case SNAT_SESSION_TCP_FIN_WAIT:
2971 case SNAT_SESSION_TCP_CLOSE_WAIT:
2972 case SNAT_SESSION_TCP_LAST_ACK:
2973 ses1->expire = now + sm->tcp_transitory_timeout;
2975 case SNAT_SESSION_TCP_ESTABLISHED:
2976 ses1->expire = now + sm->tcp_established_timeout;
2981 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2982 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2984 snat_in2out_trace_t *t =
2985 vlib_add_trace (vm, node, b1, sizeof (*t));
2986 t->is_slow_path = 0;
2987 t->sw_if_index = sw_if_index1;
2988 t->next_index = next1;
2989 t->session_index = ~0;
2991 t->session_index = ses1 - dm1->sessions;
2994 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
2996 /* verify speculative enqueues, maybe switch current next frame */
2997 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2998 to_next, n_left_to_next,
2999 bi0, bi1, next0, next1);
3002 while (n_left_from > 0 && n_left_to_next > 0)
3010 ip4_address_t new_addr0, old_addr0;
3011 u16 old_port0, new_port0, lo_port0, i0;
3012 udp_header_t * udp0;
3013 tcp_header_t * tcp0;
3015 snat_det_out_key_t key0;
3016 snat_det_map_t * dm0;
3017 snat_det_session_t * ses0 = 0;
3019 icmp46_header_t * icmp0;
3021 /* speculatively enqueue b0 to the current next frame */
3027 n_left_to_next -= 1;
3029 b0 = vlib_get_buffer (vm, bi0);
3030 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3032 ip0 = vlib_buffer_get_current (b0);
3033 udp0 = ip4_next_header (ip0);
3034 tcp0 = (tcp_header_t *) udp0;
3036 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3038 if (PREDICT_FALSE(ip0->ttl == 1))
3040 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3041 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3042 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3044 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3048 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3050 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3052 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3053 icmp0 = (icmp46_header_t *) udp0;
3055 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3056 rx_fib_index0, node, next0, thread_index,
3061 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3062 if (PREDICT_FALSE(!dm0))
3064 clib_warning("no match for internal host %U",
3065 format_ip4_address, &ip0->src_address);
3066 next0 = SNAT_IN2OUT_NEXT_DROP;
3067 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3071 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3073 key0.ext_host_addr = ip0->dst_address;
3074 key0.ext_host_port = tcp0->dst;
3076 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3077 if (PREDICT_FALSE(!ses0))
3079 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3081 key0.out_port = clib_host_to_net_u16 (lo_port0 +
3082 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3084 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3087 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3090 if (PREDICT_FALSE(!ses0))
3092 /* too many sessions for user, send ICMP error packet */
3094 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3095 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3096 ICMP4_destination_unreachable_destination_unreachable_host,
3098 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3103 new_port0 = ses0->out.out_port;
3105 old_addr0.as_u32 = ip0->src_address.as_u32;
3106 ip0->src_address.as_u32 = new_addr0.as_u32;
3107 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3109 sum0 = ip0->checksum;
3110 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3112 src_address /* changed member */);
3113 ip0->checksum = ip_csum_fold (sum0);
3115 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3117 if (tcp0->flags & TCP_FLAG_SYN)
3118 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3119 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3120 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3121 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3122 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3123 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3124 snat_det_ses_close(dm0, ses0);
3125 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3126 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3127 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3128 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3130 old_port0 = tcp0->src;
3131 tcp0->src = new_port0;
3133 sum0 = tcp0->checksum;
3134 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3136 dst_address /* changed member */);
3137 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3138 ip4_header_t /* cheat */,
3139 length /* changed member */);
3140 tcp0->checksum = ip_csum_fold(sum0);
3144 ses0->state = SNAT_SESSION_UDP_ACTIVE;
3145 old_port0 = udp0->src_port;
3146 udp0->src_port = new_port0;
3152 case SNAT_SESSION_UDP_ACTIVE:
3153 ses0->expire = now + sm->udp_timeout;
3155 case SNAT_SESSION_TCP_SYN_SENT:
3156 case SNAT_SESSION_TCP_FIN_WAIT:
3157 case SNAT_SESSION_TCP_CLOSE_WAIT:
3158 case SNAT_SESSION_TCP_LAST_ACK:
3159 ses0->expire = now + sm->tcp_transitory_timeout;
3161 case SNAT_SESSION_TCP_ESTABLISHED:
3162 ses0->expire = now + sm->tcp_established_timeout;
3167 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3168 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3170 snat_in2out_trace_t *t =
3171 vlib_add_trace (vm, node, b0, sizeof (*t));
3172 t->is_slow_path = 0;
3173 t->sw_if_index = sw_if_index0;
3174 t->next_index = next0;
3175 t->session_index = ~0;
3177 t->session_index = ses0 - dm0->sessions;
3180 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3182 /* verify speculative enqueue, maybe switch current next frame */
3183 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3184 to_next, n_left_to_next,
3188 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3191 vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3192 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3194 return frame->n_vectors;
3197 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3198 .function = snat_det_in2out_node_fn,
3199 .name = "nat44-det-in2out",
3200 .vector_size = sizeof (u32),
3201 .format_trace = format_snat_in2out_trace,
3202 .type = VLIB_NODE_TYPE_INTERNAL,
3204 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3205 .error_strings = snat_in2out_error_strings,
3207 .runtime_data_bytes = sizeof (snat_runtime_t),
3211 /* edit / add dispositions here */
3213 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3214 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3215 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3219 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3222 * Get address and port values to be used for ICMP packet translation
3223 * and create session if needed
3225 * @param[in,out] sm NAT main
3226 * @param[in,out] node NAT node runtime
3227 * @param[in] thread_index thread index
3228 * @param[in,out] b0 buffer containing packet to be translated
3229 * @param[out] p_proto protocol used for matching
3230 * @param[out] p_value address and port after NAT translation
3231 * @param[out] p_dont_translate if packet should not be translated
3232 * @param d optional parameter
3233 * @param e optional parameter
3235 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3236 u32 thread_index, vlib_buffer_t *b0,
3237 ip4_header_t *ip0, u8 *p_proto,
3238 snat_session_key_t *p_value,
3239 u8 *p_dont_translate, void *d, void *e)
3241 icmp46_header_t *icmp0;
3245 snat_det_out_key_t key0;
3246 u8 dont_translate = 0;
3248 icmp_echo_header_t *echo0, *inner_echo0 = 0;
3249 ip4_header_t *inner_ip0;
3250 void *l4_header = 0;
3251 icmp46_header_t *inner_icmp0;
3252 snat_det_map_t * dm0 = 0;
3253 ip4_address_t new_addr0;
3255 snat_det_session_t * ses0 = 0;
3256 ip4_address_t in_addr;
3259 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3260 echo0 = (icmp_echo_header_t *)(icmp0+1);
3261 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3262 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3264 if (!icmp_is_error_message (icmp0))
3266 protocol = SNAT_PROTOCOL_ICMP;
3267 in_addr = ip0->src_address;
3268 in_port = echo0->identifier;
3272 inner_ip0 = (ip4_header_t *)(echo0+1);
3273 l4_header = ip4_next_header (inner_ip0);
3274 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3275 in_addr = inner_ip0->dst_address;
3278 case SNAT_PROTOCOL_ICMP:
3279 inner_icmp0 = (icmp46_header_t*)l4_header;
3280 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3281 in_port = inner_echo0->identifier;
3283 case SNAT_PROTOCOL_UDP:
3284 case SNAT_PROTOCOL_TCP:
3285 in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3288 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3289 next0 = SNAT_IN2OUT_NEXT_DROP;
3294 dm0 = snat_det_map_by_user(sm, &in_addr);
3295 if (PREDICT_FALSE(!dm0))
3297 clib_warning("no match for internal host %U",
3298 format_ip4_address, &in_addr);
3299 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3300 IP_PROTOCOL_ICMP, rx_fib_index0)))
3305 next0 = SNAT_IN2OUT_NEXT_DROP;
3306 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3310 snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3312 key0.ext_host_addr = ip0->dst_address;
3313 key0.ext_host_port = 0;
3315 ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3316 if (PREDICT_FALSE(!ses0))
3318 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3319 IP_PROTOCOL_ICMP, rx_fib_index0)))
3324 if (icmp0->type != ICMP4_echo_request)
3326 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3327 next0 = SNAT_IN2OUT_NEXT_DROP;
3330 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3332 key0.out_port = clib_host_to_net_u16 (lo_port0 +
3333 ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3335 if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3338 ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3341 if (PREDICT_FALSE(!ses0))
3343 next0 = SNAT_IN2OUT_NEXT_DROP;
3344 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3349 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3350 !icmp_is_error_message (icmp0)))
3352 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3353 next0 = SNAT_IN2OUT_NEXT_DROP;
3357 u32 now = (u32) vlib_time_now (sm->vlib_main);
3359 ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3360 ses0->expire = now + sm->icmp_timeout;
3363 *p_proto = protocol;
3366 p_value->addr = new_addr0;
3367 p_value->fib_index = sm->outside_fib_index;
3368 p_value->port = ses0->out.out_port;
3370 *p_dont_translate = dont_translate;
3372 *(snat_det_session_t**)d = ses0;
3374 *(snat_det_map_t**)e = dm0;
3378 /**********************/
3379 /*** worker handoff ***/
3380 /**********************/
3382 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3383 vlib_node_runtime_t * node,
3384 vlib_frame_t * frame,
3387 snat_main_t *sm = &snat_main;
3388 vlib_thread_main_t *tm = vlib_get_thread_main ();
3389 u32 n_left_from, *from, *to_next = 0;
3390 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3391 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3393 vlib_frame_queue_elt_t *hf = 0;
3394 vlib_frame_t *f = 0;
3396 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3397 u32 next_worker_index = 0;
3398 u32 current_worker_index = ~0;
3399 u32 thread_index = vlib_get_thread_index ();
3403 ASSERT (vec_len (sm->workers));
3407 fq_index = sm->fq_in2out_output_index;
3408 to_node_index = sm->in2out_output_node_index;
3412 fq_index = sm->fq_in2out_index;
3413 to_node_index = sm->in2out_node_index;
3416 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3418 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3420 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3421 sm->first_worker_index + sm->num_workers - 1,
3422 (vlib_frame_queue_t *) (~0));
3425 from = vlib_frame_vector_args (frame);
3426 n_left_from = frame->n_vectors;
3428 while (n_left_from > 0)
3441 b0 = vlib_get_buffer (vm, bi0);
3443 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3444 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3446 ip0 = vlib_buffer_get_current (b0);
3448 next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3450 if (PREDICT_FALSE (next_worker_index != thread_index))
3454 if (next_worker_index != current_worker_index)
3457 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3459 hf = vlib_get_worker_handoff_queue_elt (fq_index,
3461 handoff_queue_elt_by_worker_index);
3463 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3464 to_next_worker = &hf->buffer_index[hf->n_vectors];
3465 current_worker_index = next_worker_index;
3468 /* enqueue to correct worker thread */
3469 to_next_worker[0] = bi0;
3471 n_left_to_next_worker--;
3473 if (n_left_to_next_worker == 0)
3475 hf->n_vectors = VLIB_FRAME_SIZE;
3476 vlib_put_frame_queue_elt (hf);
3477 current_worker_index = ~0;
3478 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3485 /* if this is 1st frame */
3488 f = vlib_get_frame_to_node (vm, to_node_index);
3489 to_next = vlib_frame_vector_args (f);
3497 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3498 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3500 snat_in2out_worker_handoff_trace_t *t =
3501 vlib_add_trace (vm, node, b0, sizeof (*t));
3502 t->next_worker_index = next_worker_index;
3503 t->do_handoff = do_handoff;
3508 vlib_put_frame_to_node (vm, to_node_index, f);
3511 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3513 /* Ship frames to the worker nodes */
3514 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3516 if (handoff_queue_elt_by_worker_index[i])
3518 hf = handoff_queue_elt_by_worker_index[i];
3520 * It works better to let the handoff node
3521 * rate-adapt, always ship the handoff queue element.
3523 if (1 || hf->n_vectors == hf->last_n_vectors)
3525 vlib_put_frame_queue_elt (hf);
3526 handoff_queue_elt_by_worker_index[i] = 0;
3529 hf->last_n_vectors = hf->n_vectors;
3531 congested_handoff_queue_by_worker_index[i] =
3532 (vlib_frame_queue_t *) (~0);
3535 current_worker_index = ~0;
3536 return frame->n_vectors;
3540 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3541 vlib_node_runtime_t * node,
3542 vlib_frame_t * frame)
3544 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3547 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3548 .function = snat_in2out_worker_handoff_fn,
3549 .name = "nat44-in2out-worker-handoff",
3550 .vector_size = sizeof (u32),
3551 .format_trace = format_snat_in2out_worker_handoff_trace,
3552 .type = VLIB_NODE_TYPE_INTERNAL,
3561 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3562 snat_in2out_worker_handoff_fn);
3565 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3566 vlib_node_runtime_t * node,
3567 vlib_frame_t * frame)
3569 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3572 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3573 .function = snat_in2out_output_worker_handoff_fn,
3574 .name = "nat44-in2out-output-worker-handoff",
3575 .vector_size = sizeof (u32),
3576 .format_trace = format_snat_in2out_worker_handoff_trace,
3577 .type = VLIB_NODE_TYPE_INTERNAL,
3586 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3587 snat_in2out_output_worker_handoff_fn);
3589 static_always_inline int
3590 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3592 snat_address_t * ap;
3593 clib_bihash_kv_8_8_t kv, value;
3594 snat_session_key_t m_key;
3596 vec_foreach (ap, sm->addresses)
3598 if (ap->addr.as_u32 == dst_addr->as_u32)
3602 m_key.addr.as_u32 = dst_addr->as_u32;
3603 m_key.fib_index = sm->outside_fib_index;
3606 kv.key = m_key.as_u64;
3607 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3614 snat_hairpin_dst_fn (vlib_main_t * vm,
3615 vlib_node_runtime_t * node,
3616 vlib_frame_t * frame)
3618 u32 n_left_from, * from, * to_next;
3619 snat_in2out_next_t next_index;
3620 u32 pkts_processed = 0;
3621 snat_main_t * sm = &snat_main;
3623 from = vlib_frame_vector_args (frame);
3624 n_left_from = frame->n_vectors;
3625 next_index = node->cached_next_index;
3627 while (n_left_from > 0)
3631 vlib_get_next_frame (vm, node, next_index,
3632 to_next, n_left_to_next);
3634 while (n_left_from > 0 && n_left_to_next > 0)
3642 /* speculatively enqueue b0 to the current next frame */
3648 n_left_to_next -= 1;
3650 b0 = vlib_get_buffer (vm, bi0);
3651 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3652 ip0 = vlib_buffer_get_current (b0);
3654 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3656 vnet_buffer (b0)->snat.flags = 0;
3657 if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3659 if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3661 udp_header_t * udp0 = ip4_next_header (ip0);
3662 tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3664 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3666 else if (proto0 == SNAT_PROTOCOL_ICMP)
3668 icmp46_header_t * icmp0 = ip4_next_header (ip0);
3670 snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3674 snat_hairpinning_unknown_proto (sm, b0, ip0);
3677 vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3680 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3682 /* verify speculative enqueue, maybe switch current next frame */
3683 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3684 to_next, n_left_to_next,
3688 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3691 vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3692 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3694 return frame->n_vectors;
3697 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3698 .function = snat_hairpin_dst_fn,
3699 .name = "nat44-hairpin-dst",
3700 .vector_size = sizeof (u32),
3701 .type = VLIB_NODE_TYPE_INTERNAL,
3702 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3703 .error_strings = snat_in2out_error_strings,
3706 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3707 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3711 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3712 snat_hairpin_dst_fn);
3715 snat_hairpin_src_fn (vlib_main_t * vm,
3716 vlib_node_runtime_t * node,
3717 vlib_frame_t * frame)
3719 u32 n_left_from, * from, * to_next;
3720 snat_in2out_next_t next_index;
3721 u32 pkts_processed = 0;
3722 snat_main_t *sm = &snat_main;
3724 from = vlib_frame_vector_args (frame);
3725 n_left_from = frame->n_vectors;
3726 next_index = node->cached_next_index;
3728 while (n_left_from > 0)
3732 vlib_get_next_frame (vm, node, next_index,
3733 to_next, n_left_to_next);
3735 while (n_left_from > 0 && n_left_to_next > 0)
3740 snat_interface_t *i;
3743 /* speculatively enqueue b0 to the current next frame */
3749 n_left_to_next -= 1;
3751 b0 = vlib_get_buffer (vm, bi0);
3752 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3753 next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3755 pool_foreach (i, sm->output_feature_interfaces,
3757 /* Only packets from NAT inside interface */
3758 if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3760 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3761 SNAT_FLAG_HAIRPINNING))
3763 if (PREDICT_TRUE (sm->num_workers > 1))
3764 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3766 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3772 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3774 /* verify speculative enqueue, maybe switch current next frame */
3775 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3776 to_next, n_left_to_next,
3780 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3783 vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3784 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3786 return frame->n_vectors;
3789 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3790 .function = snat_hairpin_src_fn,
3791 .name = "nat44-hairpin-src",
3792 .vector_size = sizeof (u32),
3793 .type = VLIB_NODE_TYPE_INTERNAL,
3794 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3795 .error_strings = snat_in2out_error_strings,
3796 .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3798 [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3799 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3800 [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3801 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3805 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3806 snat_hairpin_src_fn);
3809 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3810 vlib_node_runtime_t * node,
3811 vlib_frame_t * frame)
3813 u32 n_left_from, * from, * to_next;
3814 snat_in2out_next_t next_index;
3815 u32 pkts_processed = 0;
3816 snat_main_t * sm = &snat_main;
3817 u32 stats_node_index;
3819 stats_node_index = snat_in2out_fast_node.index;
3821 from = vlib_frame_vector_args (frame);
3822 n_left_from = frame->n_vectors;
3823 next_index = node->cached_next_index;
3825 while (n_left_from > 0)
3829 vlib_get_next_frame (vm, node, next_index,
3830 to_next, n_left_to_next);
3832 while (n_left_from > 0 && n_left_to_next > 0)
3840 u32 new_addr0, old_addr0;
3841 u16 old_port0, new_port0;
3842 udp_header_t * udp0;
3843 tcp_header_t * tcp0;
3844 icmp46_header_t * icmp0;
3845 snat_session_key_t key0, sm0;
3849 /* speculatively enqueue b0 to the current next frame */
3855 n_left_to_next -= 1;
3857 b0 = vlib_get_buffer (vm, bi0);
3858 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3860 ip0 = vlib_buffer_get_current (b0);
3861 udp0 = ip4_next_header (ip0);
3862 tcp0 = (tcp_header_t *) udp0;
3863 icmp0 = (icmp46_header_t *) udp0;
3865 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3866 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3868 if (PREDICT_FALSE(ip0->ttl == 1))
3870 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3871 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3872 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3874 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3878 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3880 if (PREDICT_FALSE (proto0 == ~0))
3883 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3885 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3886 rx_fib_index0, node, next0, ~0, 0, 0);
3890 key0.addr = ip0->src_address;
3891 key0.protocol = proto0;
3892 key0.port = udp0->src_port;
3893 key0.fib_index = rx_fib_index0;
3895 if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0))
3897 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3898 next0= SNAT_IN2OUT_NEXT_DROP;
3902 new_addr0 = sm0.addr.as_u32;
3903 new_port0 = sm0.port;
3904 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3905 old_addr0 = ip0->src_address.as_u32;
3906 ip0->src_address.as_u32 = new_addr0;
3908 sum0 = ip0->checksum;
3909 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3911 src_address /* changed member */);
3912 ip0->checksum = ip_csum_fold (sum0);
3914 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3916 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3918 old_port0 = tcp0->src_port;
3919 tcp0->src_port = new_port0;
3921 sum0 = tcp0->checksum;
3922 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3924 dst_address /* changed member */);
3925 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3926 ip4_header_t /* cheat */,
3927 length /* changed member */);
3928 tcp0->checksum = ip_csum_fold(sum0);
3932 old_port0 = udp0->src_port;
3933 udp0->src_port = new_port0;
3939 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3941 sum0 = tcp0->checksum;
3942 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3944 dst_address /* changed member */);
3945 tcp0->checksum = ip_csum_fold(sum0);
3950 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3953 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3954 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3956 snat_in2out_trace_t *t =
3957 vlib_add_trace (vm, node, b0, sizeof (*t));
3958 t->sw_if_index = sw_if_index0;
3959 t->next_index = next0;
3962 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3964 /* verify speculative enqueue, maybe switch current next frame */
3965 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3966 to_next, n_left_to_next,
3970 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3973 vlib_node_increment_counter (vm, stats_node_index,
3974 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3976 return frame->n_vectors;
3980 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
3981 .function = snat_in2out_fast_static_map_fn,
3982 .name = "nat44-in2out-fast",
3983 .vector_size = sizeof (u32),
3984 .format_trace = format_snat_in2out_fast_trace,
3985 .type = VLIB_NODE_TYPE_INTERNAL,
3987 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3988 .error_strings = snat_in2out_error_strings,
3990 .runtime_data_bytes = sizeof (snat_runtime_t),
3992 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3994 /* edit / add dispositions here */
3996 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3997 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3998 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
3999 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4000 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4004 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);