2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/fib/ip4_fib.h>
25 #include <nat/nat_ipfix_logging.h>
26 #include <nat/nat_det.h>
27 #include <nat/nat_reass.h>
28 #include <nat/nat_inlines.h>
30 #include <vppinfra/hash.h>
31 #include <vppinfra/error.h>
32 #include <vppinfra/elog.h>
39 } snat_in2out_trace_t;
42 u32 next_worker_index;
44 } snat_in2out_worker_handoff_trace_t;
46 /* packet trace format function */
47 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
49 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
50 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
51 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
54 tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
56 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
57 t->sw_if_index, t->next_index, t->session_index);
62 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
64 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
65 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
66 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
68 s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
69 t->sw_if_index, t->next_index);
74 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
76 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
77 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
78 snat_in2out_worker_handoff_trace_t * t =
79 va_arg (*args, snat_in2out_worker_handoff_trace_t *);
82 m = t->do_handoff ? "next worker" : "same worker";
83 s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
92 } nat44_in2out_reass_trace_t;
94 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
96 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
97 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
98 nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
100 s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
101 t->sw_if_index, t->next_index,
102 t->cached ? "cached" : "translated");
107 vlib_node_registration_t snat_in2out_node;
108 vlib_node_registration_t snat_in2out_slowpath_node;
109 vlib_node_registration_t snat_in2out_fast_node;
110 vlib_node_registration_t snat_in2out_worker_handoff_node;
111 vlib_node_registration_t snat_det_in2out_node;
112 vlib_node_registration_t snat_in2out_output_node;
113 vlib_node_registration_t snat_in2out_output_slowpath_node;
114 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
115 vlib_node_registration_t snat_hairpin_dst_node;
116 vlib_node_registration_t snat_hairpin_src_node;
117 vlib_node_registration_t nat44_hairpinning_node;
118 vlib_node_registration_t nat44_in2out_reass_node;
119 vlib_node_registration_t nat44_ed_in2out_node;
120 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
121 vlib_node_registration_t nat44_ed_in2out_output_node;
122 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
123 vlib_node_registration_t nat44_ed_hairpin_dst_node;
124 vlib_node_registration_t nat44_ed_hairpin_src_node;
125 vlib_node_registration_t nat44_ed_hairpinning_node;
127 #define foreach_snat_in2out_error \
128 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
129 _(IN2OUT_PACKETS, "Good in2out packets processed") \
130 _(OUT_OF_PORTS, "Out of ports") \
131 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \
132 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
133 _(NO_TRANSLATION, "No translation") \
134 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
135 _(DROP_FRAGMENT, "Drop fragment") \
136 _(MAX_REASS, "Maximum reassemblies exceeded") \
137 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
138 _(FQ_CONGESTED, "Handoff frame queue congested")
141 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
142 foreach_snat_in2out_error
145 } snat_in2out_error_t;
147 static char * snat_in2out_error_strings[] = {
148 #define _(sym,string) string,
149 foreach_snat_in2out_error
154 SNAT_IN2OUT_NEXT_LOOKUP,
155 SNAT_IN2OUT_NEXT_DROP,
156 SNAT_IN2OUT_NEXT_ICMP_ERROR,
157 SNAT_IN2OUT_NEXT_SLOW_PATH,
158 SNAT_IN2OUT_NEXT_REASS,
160 } snat_in2out_next_t;
163 SNAT_HAIRPIN_SRC_NEXT_DROP,
164 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
165 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
166 SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
167 SNAT_HAIRPIN_SRC_N_NEXT,
168 } snat_hairpin_next_t;
171 * @brief Check if packet should be translated
173 * Packets aimed at outside interface and external address with active session
174 * should be translated.
177 * @param rt NAT runtime data
178 * @param sw_if_index0 index of the inside interface
179 * @param ip0 IPv4 header
180 * @param proto0 NAT protocol
181 * @param rx_fib_index0 RX FIB index
183 * @returns 0 if packet should be translated otherwise 1
186 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
187 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
193 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
194 nat_outside_fib_t *outside_fib;
196 .fp_proto = FIB_PROTOCOL_IP4,
199 .ip4.as_u32 = ip0->dst_address.as_u32,
203 /* Don't NAT packet aimed at the intfc address */
204 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
205 ip0->dst_address.as_u32)))
208 fei = fib_table_lookup (rx_fib_index0, &pfx);
209 if (FIB_NODE_INDEX_INVALID != fei)
211 u32 sw_if_index = fib_entry_get_resolving_interface (fei);
212 if (sw_if_index == ~0)
214 vec_foreach (outside_fib, sm->outside_fibs)
216 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
217 if (FIB_NODE_INDEX_INVALID != fei)
219 sw_if_index = fib_entry_get_resolving_interface (fei);
220 if (sw_if_index != ~0)
225 if (sw_if_index == ~0)
229 pool_foreach (i, sm->interfaces,
231 /* NAT packet aimed at outside interface */
232 if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
241 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
242 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
243 u32 rx_fib_index0, u32 thread_index)
245 udp_header_t * udp0 = ip4_next_header (ip0);
246 snat_session_key_t key0, sm0;
247 clib_bihash_kv_8_8_t kv0, value0;
249 key0.addr = ip0->dst_address;
250 key0.port = udp0->dst_port;
251 key0.protocol = proto0;
252 key0.fib_index = sm->outside_fib_index;
253 kv0.key = key0.as_u64;
255 /* NAT packet aimed at external address if */
256 /* has active sessions */
257 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
260 /* or is static mappings */
261 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
267 if (sm->forwarding_enabled)
270 return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
275 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
276 u32 proto0, u16 src_port, u16 dst_port,
277 u32 thread_index, u32 sw_if_index)
279 snat_session_key_t key0;
280 clib_bihash_kv_8_8_t kv0, value0;
284 key0.addr = ip0->src_address;
285 key0.port = src_port;
286 key0.protocol = proto0;
287 key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
288 kv0.key = key0.as_u64;
290 if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
295 key0.addr = ip0->dst_address;
296 key0.port = dst_port;
297 key0.protocol = proto0;
298 kv0.key = key0.as_u64;
299 if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
303 pool_foreach (i, sm->output_feature_interfaces,
305 if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
314 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
317 snat_session_key_t * key0,
318 snat_session_t ** sessionp,
319 vlib_node_runtime_t * node,
325 clib_bihash_kv_8_8_t kv0;
326 snat_session_key_t key1;
327 u32 address_index = ~0;
328 udp_header_t * udp0 = ip4_next_header (ip0);
330 nat_outside_fib_t *outside_fib;
331 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
333 .fp_proto = FIB_PROTOCOL_IP4,
336 .ip4.as_u32 = ip0->dst_address.as_u32,
340 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
342 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
343 nat_ipfix_logging_max_sessions(sm->max_translations);
344 nat_log_notice ("maximum sessions exceeded");
345 return SNAT_IN2OUT_NEXT_DROP;
348 key1.protocol = key0->protocol;
350 /* First try to match static mapping by local address and port */
351 if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
353 /* Try to create dynamic translation */
354 if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
358 sm->per_thread_data[thread_index].snat_thread_index))
360 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
361 return SNAT_IN2OUT_NEXT_DROP;
367 u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
371 nat_log_warn ("create NAT user failed");
372 return SNAT_IN2OUT_NEXT_DROP;
375 s = nat_session_alloc_or_recycle (sm, u, thread_index);
378 nat44_delete_user_with_no_session (sm, u, thread_index);
379 nat_log_warn ("create NAT session failed");
380 return SNAT_IN2OUT_NEXT_DROP;
384 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
385 user_session_increment (sm, u, is_sm);
386 s->outside_address_index = address_index;
389 s->out2in.protocol = key0->protocol;
390 s->out2in.fib_index = sm->outside_fib_index;
391 switch (vec_len (sm->outside_fibs))
394 s->out2in.fib_index = sm->outside_fib_index;
397 s->out2in.fib_index = sm->outside_fibs[0].fib_index;
400 vec_foreach (outside_fib, sm->outside_fibs)
402 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
403 if (FIB_NODE_INDEX_INVALID != fei)
405 if (fib_entry_get_resolving_interface (fei) != ~0)
407 s->out2in.fib_index = outside_fib->fib_index;
414 s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
415 s->ext_host_port = udp0->dst_port;
418 /* Add to translation hashes */
419 kv0.key = s->in2out.as_u64;
420 kv0.value = s - sm->per_thread_data[thread_index].sessions;
421 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
423 nat_log_notice ("in2out key add failed");
425 kv0.key = s->out2in.as_u64;
426 kv0.value = s - sm->per_thread_data[thread_index].sessions;
428 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
430 nat_log_notice ("out2in key add failed");
433 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
434 s->out2in.addr.as_u32,
438 s->in2out.fib_index);
443 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
444 snat_session_key_t *p_key0)
446 icmp46_header_t *icmp0;
447 snat_session_key_t key0;
448 icmp_echo_header_t *echo0, *inner_echo0 = 0;
449 ip4_header_t *inner_ip0 = 0;
451 icmp46_header_t *inner_icmp0;
453 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
454 echo0 = (icmp_echo_header_t *)(icmp0+1);
456 if (!icmp_is_error_message (icmp0))
458 key0.protocol = SNAT_PROTOCOL_ICMP;
459 key0.addr = ip0->src_address;
460 key0.port = echo0->identifier;
464 inner_ip0 = (ip4_header_t *)(echo0+1);
465 l4_header = ip4_next_header (inner_ip0);
466 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
467 key0.addr = inner_ip0->dst_address;
468 switch (key0.protocol)
470 case SNAT_PROTOCOL_ICMP:
471 inner_icmp0 = (icmp46_header_t*)l4_header;
472 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
473 key0.port = inner_echo0->identifier;
475 case SNAT_PROTOCOL_UDP:
476 case SNAT_PROTOCOL_TCP:
477 key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
480 return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
484 return -1; /* success */
488 * Get address and port values to be used for ICMP packet translation
489 * and create session if needed
491 * @param[in,out] sm NAT main
492 * @param[in,out] node NAT node runtime
493 * @param[in] thread_index thread index
494 * @param[in,out] b0 buffer containing packet to be translated
495 * @param[out] p_proto protocol used for matching
496 * @param[out] p_value address and port after NAT translation
497 * @param[out] p_dont_translate if packet should not be translated
498 * @param d optional parameter
499 * @param e optional parameter
501 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
502 u32 thread_index, vlib_buffer_t *b0,
503 ip4_header_t *ip0, u8 *p_proto,
504 snat_session_key_t *p_value,
505 u8 *p_dont_translate, void *d, void *e)
507 icmp46_header_t *icmp0;
510 snat_session_key_t key0;
511 snat_session_t *s0 = 0;
512 u8 dont_translate = 0;
513 clib_bihash_kv_8_8_t kv0, value0;
517 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
518 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
519 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
521 err = icmp_get_key (ip0, &key0);
524 b0->error = node->errors[err];
525 next0 = SNAT_IN2OUT_NEXT_DROP;
528 key0.fib_index = rx_fib_index0;
530 kv0.key = key0.as_u64;
532 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
535 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
537 if (PREDICT_FALSE(nat_not_translate_output_feature(sm, ip0,
538 key0.protocol, key0.port, key0.port, thread_index, sw_if_index0)))
546 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
547 ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
554 if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
556 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
557 next0 = SNAT_IN2OUT_NEXT_DROP;
561 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
562 &s0, node, next0, thread_index);
564 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
569 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
570 icmp0->type != ICMP4_echo_reply &&
571 !icmp_is_error_message (icmp0)))
573 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
574 next0 = SNAT_IN2OUT_NEXT_DROP;
578 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
583 *p_proto = key0.protocol;
585 *p_value = s0->out2in;
586 *p_dont_translate = dont_translate;
588 *(snat_session_t**)d = s0;
593 * Get address and port values to be used for ICMP packet translation
595 * @param[in] sm NAT main
596 * @param[in,out] node NAT node runtime
597 * @param[in] thread_index thread index
598 * @param[in,out] b0 buffer containing packet to be translated
599 * @param[out] p_proto protocol used for matching
600 * @param[out] p_value address and port after NAT translation
601 * @param[out] p_dont_translate if packet should not be translated
602 * @param d optional parameter
603 * @param e optional parameter
605 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
606 u32 thread_index, vlib_buffer_t *b0,
607 ip4_header_t *ip0, u8 *p_proto,
608 snat_session_key_t *p_value,
609 u8 *p_dont_translate, void *d, void *e)
611 icmp46_header_t *icmp0;
614 snat_session_key_t key0;
615 snat_session_key_t sm0;
616 u8 dont_translate = 0;
621 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
622 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
623 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
625 err = icmp_get_key (ip0, &key0);
628 b0->error = node->errors[err];
629 next0 = SNAT_IN2OUT_NEXT_DROP;
632 key0.fib_index = rx_fib_index0;
634 if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
636 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
637 IP_PROTOCOL_ICMP, rx_fib_index0)))
643 if (icmp_is_error_message (icmp0))
645 next0 = SNAT_IN2OUT_NEXT_DROP;
649 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
650 next0 = SNAT_IN2OUT_NEXT_DROP;
654 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
655 (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
656 !icmp_is_error_message (icmp0)))
658 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
659 next0 = SNAT_IN2OUT_NEXT_DROP;
666 *p_proto = key0.protocol;
667 *p_dont_translate = dont_translate;
671 static inline u32 icmp_in2out (snat_main_t *sm,
674 icmp46_header_t * icmp0,
677 vlib_node_runtime_t * node,
683 snat_session_key_t sm0;
685 icmp_echo_header_t *echo0, *inner_echo0 = 0;
686 ip4_header_t *inner_ip0;
688 icmp46_header_t *inner_icmp0;
690 u32 new_addr0, old_addr0;
691 u16 old_id0, new_id0;
696 echo0 = (icmp_echo_header_t *)(icmp0+1);
698 next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
699 &protocol, &sm0, &dont_translate, d, e);
702 if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
705 sum0 = ip_incremental_checksum (0, icmp0,
706 ntohs(ip0->length) - ip4_header_bytes (ip0));
707 checksum0 = ~ip_csum_fold (sum0);
708 if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
710 next0 = SNAT_IN2OUT_NEXT_DROP;
714 old_addr0 = ip0->src_address.as_u32;
715 new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
716 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
717 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
719 sum0 = ip0->checksum;
720 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
721 src_address /* changed member */);
722 ip0->checksum = ip_csum_fold (sum0);
724 if (icmp0->checksum == 0)
725 icmp0->checksum = 0xffff;
727 if (!icmp_is_error_message (icmp0))
730 if (PREDICT_FALSE(new_id0 != echo0->identifier))
732 old_id0 = echo0->identifier;
734 echo0->identifier = new_id0;
736 sum0 = icmp0->checksum;
737 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
739 icmp0->checksum = ip_csum_fold (sum0);
744 inner_ip0 = (ip4_header_t *)(echo0+1);
745 l4_header = ip4_next_header (inner_ip0);
747 if (!ip4_header_checksum_is_valid (inner_ip0))
749 next0 = SNAT_IN2OUT_NEXT_DROP;
753 old_addr0 = inner_ip0->dst_address.as_u32;
754 inner_ip0->dst_address = sm0.addr;
755 new_addr0 = inner_ip0->dst_address.as_u32;
757 sum0 = icmp0->checksum;
758 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
759 dst_address /* changed member */);
760 icmp0->checksum = ip_csum_fold (sum0);
764 case SNAT_PROTOCOL_ICMP:
765 inner_icmp0 = (icmp46_header_t*)l4_header;
766 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
768 old_id0 = inner_echo0->identifier;
770 inner_echo0->identifier = new_id0;
772 sum0 = icmp0->checksum;
773 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
775 icmp0->checksum = ip_csum_fold (sum0);
777 case SNAT_PROTOCOL_UDP:
778 case SNAT_PROTOCOL_TCP:
779 old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
781 ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
783 sum0 = icmp0->checksum;
784 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
786 icmp0->checksum = ip_csum_fold (sum0);
800 * Hairpinning allows two endpoints on the internal side of the NAT to
801 * communicate even if they only use each other's external IP addresses
804 * @param sm NAT main.
805 * @param b0 Vlib buffer.
806 * @param ip0 IP header.
807 * @param udp0 UDP header.
808 * @param tcp0 TCP header.
809 * @param proto0 NAT protocol.
812 snat_hairpinning (snat_main_t *sm,
820 snat_session_key_t key0, sm0;
822 clib_bihash_kv_8_8_t kv0, value0;
824 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
825 u16 new_dst_port0, old_dst_port0;
828 key0.addr = ip0->dst_address;
829 key0.port = udp0->dst_port;
830 key0.protocol = proto0;
831 key0.fib_index = sm->outside_fib_index;
832 kv0.key = key0.as_u64;
834 /* Check if destination is static mappings */
835 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
837 new_dst_addr0 = sm0.addr.as_u32;
838 new_dst_port0 = sm0.port;
839 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
841 /* or active session */
844 if (sm->num_workers > 1)
845 ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
847 ti = sm->num_workers;
851 clib_bihash_kv_16_8_t ed_kv, ed_value;
852 make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
853 ip0->protocol, sm->outside_fib_index, udp0->dst_port,
855 rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
861 rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
868 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
869 new_dst_addr0 = s0->in2out.addr.as_u32;
870 new_dst_port0 = s0->in2out.port;
871 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
874 /* Destination is behind the same NAT, use internal address and port */
877 old_dst_addr0 = ip0->dst_address.as_u32;
878 ip0->dst_address.as_u32 = new_dst_addr0;
879 sum0 = ip0->checksum;
880 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
881 ip4_header_t, dst_address);
882 ip0->checksum = ip_csum_fold (sum0);
884 old_dst_port0 = tcp0->dst;
885 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
887 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
889 tcp0->dst = new_dst_port0;
890 sum0 = tcp0->checksum;
891 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
892 ip4_header_t, dst_address);
893 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
894 ip4_header_t /* cheat */, length);
895 tcp0->checksum = ip_csum_fold(sum0);
899 udp0->dst_port = new_dst_port0;
905 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
907 sum0 = tcp0->checksum;
908 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
909 ip4_header_t, dst_address);
910 tcp0->checksum = ip_csum_fold(sum0);
919 snat_icmp_hairpinning (snat_main_t *sm,
922 icmp46_header_t * icmp0,
925 snat_session_key_t key0, sm0;
926 clib_bihash_kv_8_8_t kv0, value0;
927 u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
932 if (!icmp_is_error_message (icmp0))
934 icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
935 u16 icmp_id0 = echo0->identifier;
936 key0.addr = ip0->dst_address;
937 key0.port = icmp_id0;
938 key0.protocol = SNAT_PROTOCOL_ICMP;
939 key0.fib_index = sm->outside_fib_index;
940 kv0.key = key0.as_u64;
942 if (sm->num_workers > 1)
943 ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
945 ti = sm->num_workers;
947 /* Check if destination is in active sessions */
950 clib_bihash_kv_16_8_t ed_kv, ed_value;
951 make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
952 IP_PROTOCOL_ICMP, sm->outside_fib_index, icmp_id0, 0);
953 rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
959 rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
965 /* or static mappings */
966 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
968 new_dst_addr0 = sm0.addr.as_u32;
969 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
974 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
975 new_dst_addr0 = s0->in2out.addr.as_u32;
976 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
977 echo0->identifier = s0->in2out.port;
978 sum0 = icmp0->checksum;
979 sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
980 icmp_echo_header_t, identifier);
981 icmp0->checksum = ip_csum_fold (sum0);
984 /* Destination is behind the same NAT, use internal address and port */
987 old_dst_addr0 = ip0->dst_address.as_u32;
988 ip0->dst_address.as_u32 = new_dst_addr0;
989 sum0 = ip0->checksum;
990 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
991 ip4_header_t, dst_address);
992 ip0->checksum = ip_csum_fold (sum0);
998 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1001 icmp46_header_t * icmp0,
1004 vlib_node_runtime_t * node,
1008 snat_session_t ** p_s0)
1010 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1011 next0, thread_index, p_s0, 0);
1012 snat_session_t * s0 = *p_s0;
1013 if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1016 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1017 snat_icmp_hairpinning(sm, b0, ip0, icmp0, sm->endpoint_dependent);
1019 nat44_session_update_counters (s0, now,
1020 vlib_buffer_length_in_chain (sm->vlib_main, b0));
1021 /* Per-user LRU list maintenance */
1022 nat44_session_update_lru (sm, s0, thread_index);
1028 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
1032 clib_bihash_kv_8_8_t kv, value;
1033 snat_static_mapping_t *m;
1034 u32 old_addr, new_addr;
1037 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1038 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1041 m = pool_elt_at_index (sm->static_mappings, value.value);
1043 old_addr = ip->dst_address.as_u32;
1044 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1046 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1047 ip->checksum = ip_csum_fold (sum);
1049 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1050 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1054 nat_in2out_sm_unknown_proto (snat_main_t *sm,
1059 clib_bihash_kv_8_8_t kv, value;
1060 snat_static_mapping_t *m;
1061 snat_session_key_t m_key;
1062 u32 old_addr, new_addr;
1065 m_key.addr = ip->src_address;
1068 m_key.fib_index = rx_fib_index;
1069 kv.key = m_key.as_u64;
1070 if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1073 m = pool_elt_at_index (sm->static_mappings, value.value);
1075 old_addr = ip->src_address.as_u32;
1076 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1078 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1079 ip->checksum = ip_csum_fold (sum);
1083 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1085 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1086 nat_hairpinning_sm_unknown_proto (sm, b, ip);
1093 snat_in2out_node_fn_inline (vlib_main_t * vm,
1094 vlib_node_runtime_t * node,
1095 vlib_frame_t * frame, int is_slow_path,
1096 int is_output_feature)
1098 u32 n_left_from, * from, * to_next;
1099 snat_in2out_next_t next_index;
1100 u32 pkts_processed = 0;
1101 snat_main_t * sm = &snat_main;
1102 f64 now = vlib_time_now (vm);
1103 u32 stats_node_index;
1104 u32 thread_index = vm->thread_index;
1106 stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1107 snat_in2out_node.index;
1109 from = vlib_frame_vector_args (frame);
1110 n_left_from = frame->n_vectors;
1111 next_index = node->cached_next_index;
1113 while (n_left_from > 0)
1117 vlib_get_next_frame (vm, node, next_index,
1118 to_next, n_left_to_next);
1120 while (n_left_from >= 4 && n_left_to_next >= 2)
1123 vlib_buffer_t * b0, * b1;
1125 u32 sw_if_index0, sw_if_index1;
1126 ip4_header_t * ip0, * ip1;
1127 ip_csum_t sum0, sum1;
1128 u32 new_addr0, old_addr0, new_addr1, old_addr1;
1129 u16 old_port0, new_port0, old_port1, new_port1;
1130 udp_header_t * udp0, * udp1;
1131 tcp_header_t * tcp0, * tcp1;
1132 icmp46_header_t * icmp0, * icmp1;
1133 snat_session_key_t key0, key1;
1134 u32 rx_fib_index0, rx_fib_index1;
1136 snat_session_t * s0 = 0, * s1 = 0;
1137 clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1138 u32 iph_offset0 = 0, iph_offset1 = 0;
1140 /* Prefetch next iteration. */
1142 vlib_buffer_t * p2, * p3;
1144 p2 = vlib_get_buffer (vm, from[2]);
1145 p3 = vlib_get_buffer (vm, from[3]);
1147 vlib_prefetch_buffer_header (p2, LOAD);
1148 vlib_prefetch_buffer_header (p3, LOAD);
1150 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1151 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1154 /* speculatively enqueue b0 and b1 to the current next frame */
1155 to_next[0] = bi0 = from[0];
1156 to_next[1] = bi1 = from[1];
1160 n_left_to_next -= 2;
1162 b0 = vlib_get_buffer (vm, bi0);
1163 b1 = vlib_get_buffer (vm, bi1);
1165 if (is_output_feature)
1166 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1168 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1171 udp0 = ip4_next_header (ip0);
1172 tcp0 = (tcp_header_t *) udp0;
1173 icmp0 = (icmp46_header_t *) udp0;
1175 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1176 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1179 next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1181 if (PREDICT_FALSE(ip0->ttl == 1))
1183 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1184 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1185 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1187 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1191 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1193 /* Next configured feature, probably ip4-lookup */
1196 if (PREDICT_FALSE (proto0 == ~0))
1198 if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1200 next0 = SNAT_IN2OUT_NEXT_DROP;
1201 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1206 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1208 next0 = icmp_in2out_slow_path
1209 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1210 node, next0, now, thread_index, &s0);
1216 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1218 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1222 if (ip4_is_fragment (ip0))
1224 next0 = SNAT_IN2OUT_NEXT_REASS;
1229 key0.addr = ip0->src_address;
1230 key0.port = udp0->src_port;
1231 key0.protocol = proto0;
1232 key0.fib_index = rx_fib_index0;
1234 kv0.key = key0.as_u64;
1236 if (PREDICT_FALSE (clib_bihash_search_8_8 (
1237 &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1241 if (is_output_feature)
1243 if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1244 ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1249 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1250 ip0, proto0, rx_fib_index0, thread_index)))
1254 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1255 &s0, node, next0, thread_index);
1256 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1261 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1266 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1269 b0->flags |= VNET_BUFFER_F_IS_NATED;
1271 old_addr0 = ip0->src_address.as_u32;
1272 ip0->src_address = s0->out2in.addr;
1273 new_addr0 = ip0->src_address.as_u32;
1274 if (!is_output_feature)
1275 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1277 sum0 = ip0->checksum;
1278 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1280 src_address /* changed member */);
1281 ip0->checksum = ip_csum_fold (sum0);
1283 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1285 old_port0 = tcp0->src_port;
1286 tcp0->src_port = s0->out2in.port;
1287 new_port0 = tcp0->src_port;
1289 sum0 = tcp0->checksum;
1290 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1292 dst_address /* changed member */);
1293 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1294 ip4_header_t /* cheat */,
1295 length /* changed member */);
1296 tcp0->checksum = ip_csum_fold(sum0);
1300 old_port0 = udp0->src_port;
1301 udp0->src_port = s0->out2in.port;
1306 nat44_session_update_counters (s0, now,
1307 vlib_buffer_length_in_chain (vm, b0));
1308 /* Per-user LRU list maintenance */
1309 nat44_session_update_lru (sm, s0, thread_index);
1312 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1313 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1315 snat_in2out_trace_t *t =
1316 vlib_add_trace (vm, node, b0, sizeof (*t));
1317 t->is_slow_path = is_slow_path;
1318 t->sw_if_index = sw_if_index0;
1319 t->next_index = next0;
1320 t->session_index = ~0;
1322 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1325 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1327 if (is_output_feature)
1328 iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1330 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1333 udp1 = ip4_next_header (ip1);
1334 tcp1 = (tcp_header_t *) udp1;
1335 icmp1 = (icmp46_header_t *) udp1;
1337 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1338 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1341 if (PREDICT_FALSE(ip1->ttl == 1))
1343 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1344 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1345 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1347 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1351 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1353 /* Next configured feature, probably ip4-lookup */
1356 if (PREDICT_FALSE (proto1 == ~0))
1358 if (nat_in2out_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1360 next1 = SNAT_IN2OUT_NEXT_DROP;
1361 b1->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1366 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1368 next1 = icmp_in2out_slow_path
1369 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1370 next1, now, thread_index, &s1);
1376 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1378 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1382 if (ip4_is_fragment (ip1))
1384 next1 = SNAT_IN2OUT_NEXT_REASS;
1389 key1.addr = ip1->src_address;
1390 key1.port = udp1->src_port;
1391 key1.protocol = proto1;
1392 key1.fib_index = rx_fib_index1;
1394 kv1.key = key1.as_u64;
1396 if (PREDICT_FALSE(clib_bihash_search_8_8 (
1397 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1401 if (is_output_feature)
1403 if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1404 ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1409 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1410 ip1, proto1, rx_fib_index1, thread_index)))
1414 next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1415 &s1, node, next1, thread_index);
1416 if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1421 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1426 s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1429 b1->flags |= VNET_BUFFER_F_IS_NATED;
1431 old_addr1 = ip1->src_address.as_u32;
1432 ip1->src_address = s1->out2in.addr;
1433 new_addr1 = ip1->src_address.as_u32;
1434 if (!is_output_feature)
1435 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1437 sum1 = ip1->checksum;
1438 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1440 src_address /* changed member */);
1441 ip1->checksum = ip_csum_fold (sum1);
1443 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1445 old_port1 = tcp1->src_port;
1446 tcp1->src_port = s1->out2in.port;
1447 new_port1 = tcp1->src_port;
1449 sum1 = tcp1->checksum;
1450 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1452 dst_address /* changed member */);
1453 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1454 ip4_header_t /* cheat */,
1455 length /* changed member */);
1456 tcp1->checksum = ip_csum_fold(sum1);
1460 old_port1 = udp1->src_port;
1461 udp1->src_port = s1->out2in.port;
1466 nat44_session_update_counters (s1, now,
1467 vlib_buffer_length_in_chain (vm, b1));
1468 /* Per-user LRU list maintenance */
1469 nat44_session_update_lru (sm, s1, thread_index);
1472 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1473 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1475 snat_in2out_trace_t *t =
1476 vlib_add_trace (vm, node, b1, sizeof (*t));
1477 t->sw_if_index = sw_if_index1;
1478 t->next_index = next1;
1479 t->session_index = ~0;
1481 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1484 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1486 /* verify speculative enqueues, maybe switch current next frame */
1487 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1488 to_next, n_left_to_next,
1489 bi0, bi1, next0, next1);
1492 while (n_left_from > 0 && n_left_to_next > 0)
1500 u32 new_addr0, old_addr0;
1501 u16 old_port0, new_port0;
1502 udp_header_t * udp0;
1503 tcp_header_t * tcp0;
1504 icmp46_header_t * icmp0;
1505 snat_session_key_t key0;
1508 snat_session_t * s0 = 0;
1509 clib_bihash_kv_8_8_t kv0, value0;
1510 u32 iph_offset0 = 0;
1512 /* speculatively enqueue b0 to the current next frame */
1518 n_left_to_next -= 1;
1520 b0 = vlib_get_buffer (vm, bi0);
1521 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1523 if (is_output_feature)
1524 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1526 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1529 udp0 = ip4_next_header (ip0);
1530 tcp0 = (tcp_header_t *) udp0;
1531 icmp0 = (icmp46_header_t *) udp0;
1533 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1534 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1537 if (PREDICT_FALSE(ip0->ttl == 1))
1539 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1540 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1541 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1543 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1547 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1549 /* Next configured feature, probably ip4-lookup */
1552 if (PREDICT_FALSE (proto0 == ~0))
1554 if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1556 next0 = SNAT_IN2OUT_NEXT_DROP;
1557 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1562 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1564 next0 = icmp_in2out_slow_path
1565 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1566 next0, now, thread_index, &s0);
1572 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1574 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1578 if (ip4_is_fragment (ip0))
1580 next0 = SNAT_IN2OUT_NEXT_REASS;
1585 key0.addr = ip0->src_address;
1586 key0.port = udp0->src_port;
1587 key0.protocol = proto0;
1588 key0.fib_index = rx_fib_index0;
1590 kv0.key = key0.as_u64;
1592 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1597 if (is_output_feature)
1599 if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1600 ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1605 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1606 ip0, proto0, rx_fib_index0, thread_index)))
1610 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1611 &s0, node, next0, thread_index);
1613 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1618 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1623 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1626 b0->flags |= VNET_BUFFER_F_IS_NATED;
1628 old_addr0 = ip0->src_address.as_u32;
1629 ip0->src_address = s0->out2in.addr;
1630 new_addr0 = ip0->src_address.as_u32;
1631 if (!is_output_feature)
1632 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1634 sum0 = ip0->checksum;
1635 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1637 src_address /* changed member */);
1638 ip0->checksum = ip_csum_fold (sum0);
1640 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1642 old_port0 = tcp0->src_port;
1643 tcp0->src_port = s0->out2in.port;
1644 new_port0 = tcp0->src_port;
1646 sum0 = tcp0->checksum;
1647 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1649 dst_address /* changed member */);
1650 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1651 ip4_header_t /* cheat */,
1652 length /* changed member */);
1653 tcp0->checksum = ip_csum_fold(sum0);
1657 old_port0 = udp0->src_port;
1658 udp0->src_port = s0->out2in.port;
1663 nat44_session_update_counters (s0, now,
1664 vlib_buffer_length_in_chain (vm, b0));
1665 /* Per-user LRU list maintenance */
1666 nat44_session_update_lru (sm, s0, thread_index);
1669 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1670 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1672 snat_in2out_trace_t *t =
1673 vlib_add_trace (vm, node, b0, sizeof (*t));
1674 t->is_slow_path = is_slow_path;
1675 t->sw_if_index = sw_if_index0;
1676 t->next_index = next0;
1677 t->session_index = ~0;
1679 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1682 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1684 /* verify speculative enqueue, maybe switch current next frame */
1685 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1686 to_next, n_left_to_next,
1690 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1693 vlib_node_increment_counter (vm, stats_node_index,
1694 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1696 return frame->n_vectors;
1700 snat_in2out_fast_path_fn (vlib_main_t * vm,
1701 vlib_node_runtime_t * node,
1702 vlib_frame_t * frame)
1704 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1707 VLIB_REGISTER_NODE (snat_in2out_node) = {
1708 .function = snat_in2out_fast_path_fn,
1709 .name = "nat44-in2out",
1710 .vector_size = sizeof (u32),
1711 .format_trace = format_snat_in2out_trace,
1712 .type = VLIB_NODE_TYPE_INTERNAL,
1714 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1715 .error_strings = snat_in2out_error_strings,
1717 .runtime_data_bytes = sizeof (snat_runtime_t),
1719 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1721 /* edit / add dispositions here */
1723 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1724 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1725 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1726 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1727 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1731 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1734 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1735 vlib_node_runtime_t * node,
1736 vlib_frame_t * frame)
1738 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1741 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1742 .function = snat_in2out_output_fast_path_fn,
1743 .name = "nat44-in2out-output",
1744 .vector_size = sizeof (u32),
1745 .format_trace = format_snat_in2out_trace,
1746 .type = VLIB_NODE_TYPE_INTERNAL,
1748 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1749 .error_strings = snat_in2out_error_strings,
1751 .runtime_data_bytes = sizeof (snat_runtime_t),
1753 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1755 /* edit / add dispositions here */
1757 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1758 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1759 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1760 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1761 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1765 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1766 snat_in2out_output_fast_path_fn);
1769 snat_in2out_slow_path_fn (vlib_main_t * vm,
1770 vlib_node_runtime_t * node,
1771 vlib_frame_t * frame)
1773 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
1776 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1777 .function = snat_in2out_slow_path_fn,
1778 .name = "nat44-in2out-slowpath",
1779 .vector_size = sizeof (u32),
1780 .format_trace = format_snat_in2out_trace,
1781 .type = VLIB_NODE_TYPE_INTERNAL,
1783 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1784 .error_strings = snat_in2out_error_strings,
1786 .runtime_data_bytes = sizeof (snat_runtime_t),
1788 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1790 /* edit / add dispositions here */
1792 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1793 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1794 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1795 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1796 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1800 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1801 snat_in2out_slow_path_fn);
1804 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1805 vlib_node_runtime_t * node,
1806 vlib_frame_t * frame)
1808 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
1811 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1812 .function = snat_in2out_output_slow_path_fn,
1813 .name = "nat44-in2out-output-slowpath",
1814 .vector_size = sizeof (u32),
1815 .format_trace = format_snat_in2out_trace,
1816 .type = VLIB_NODE_TYPE_INTERNAL,
1818 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1819 .error_strings = snat_in2out_error_strings,
1821 .runtime_data_bytes = sizeof (snat_runtime_t),
1823 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1825 /* edit / add dispositions here */
1827 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1828 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1829 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1830 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1831 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1835 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1836 snat_in2out_output_slow_path_fn);
1838 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
1841 nat44_hairpinning_fn_inline (vlib_main_t * vm,
1842 vlib_node_runtime_t * node,
1843 vlib_frame_t * frame,
1846 u32 n_left_from, * from, * to_next, stats_node_index;
1847 snat_in2out_next_t next_index;
1848 u32 pkts_processed = 0;
1849 snat_main_t * sm = &snat_main;
1850 vnet_feature_main_t *fm = &feature_main;
1851 u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1852 vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
1854 stats_node_index = is_ed ? nat44_ed_hairpinning_node.index :
1855 nat44_hairpinning_node.index;
1856 from = vlib_frame_vector_args (frame);
1857 n_left_from = frame->n_vectors;
1858 next_index = node->cached_next_index;
1860 while (n_left_from > 0)
1864 vlib_get_next_frame (vm, node, next_index,
1865 to_next, n_left_to_next);
1867 while (n_left_from > 0 && n_left_to_next > 0)
1874 udp_header_t * udp0;
1875 tcp_header_t * tcp0;
1877 /* speculatively enqueue b0 to the current next frame */
1883 n_left_to_next -= 1;
1885 b0 = vlib_get_buffer (vm, bi0);
1886 ip0 = vlib_buffer_get_current (b0);
1887 udp0 = ip4_next_header (ip0);
1888 tcp0 = (tcp_header_t *) udp0;
1890 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1892 vnet_get_config_data (&cm->config_main, &b0->current_config_index,
1895 if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
1896 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1898 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1900 /* verify speculative enqueue, maybe switch current next frame */
1901 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1902 to_next, n_left_to_next,
1906 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1909 vlib_node_increment_counter (vm, stats_node_index,
1910 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1912 return frame->n_vectors;
1916 nat44_hairpinning_fn (vlib_main_t * vm,
1917 vlib_node_runtime_t * node,
1918 vlib_frame_t * frame)
1920 return nat44_hairpinning_fn_inline (vm, node, frame, 0);
1923 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
1924 .function = nat44_hairpinning_fn,
1925 .name = "nat44-hairpinning",
1926 .vector_size = sizeof (u32),
1927 .type = VLIB_NODE_TYPE_INTERNAL,
1928 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1929 .error_strings = snat_in2out_error_strings,
1932 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1933 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1937 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
1938 nat44_hairpinning_fn);
1941 nat44_ed_hairpinning_fn (vlib_main_t * vm,
1942 vlib_node_runtime_t * node,
1943 vlib_frame_t * frame)
1945 return nat44_hairpinning_fn_inline (vm, node, frame, 1);
1948 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
1949 .function = nat44_ed_hairpinning_fn,
1950 .name = "nat44-ed-hairpinning",
1951 .vector_size = sizeof (u32),
1952 .type = VLIB_NODE_TYPE_INTERNAL,
1953 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1954 .error_strings = snat_in2out_error_strings,
1957 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1958 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1962 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpinning_node,
1963 nat44_ed_hairpinning_fn);
1966 nat44_reass_hairpinning (snat_main_t *sm,
1973 snat_session_key_t key0, sm0;
1974 snat_session_t * s0;
1975 clib_bihash_kv_8_8_t kv0, value0;
1977 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
1978 u16 new_dst_port0, old_dst_port0;
1979 udp_header_t * udp0;
1980 tcp_header_t * tcp0;
1982 key0.addr = ip0->dst_address;
1984 key0.protocol = proto0;
1985 key0.fib_index = sm->outside_fib_index;
1986 kv0.key = key0.as_u64;
1988 udp0 = ip4_next_header (ip0);
1990 /* Check if destination is static mappings */
1991 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1993 new_dst_addr0 = sm0.addr.as_u32;
1994 new_dst_port0 = sm0.port;
1995 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1997 /* or active sessions */
2000 if (sm->num_workers > 1)
2001 ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2003 ti = sm->num_workers;
2005 if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2008 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2009 new_dst_addr0 = s0->in2out.addr.as_u32;
2010 new_dst_port0 = s0->in2out.port;
2011 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2015 /* Destination is behind the same NAT, use internal address and port */
2018 old_dst_addr0 = ip0->dst_address.as_u32;
2019 ip0->dst_address.as_u32 = new_dst_addr0;
2020 sum0 = ip0->checksum;
2021 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2022 ip4_header_t, dst_address);
2023 ip0->checksum = ip_csum_fold (sum0);
2025 old_dst_port0 = dport;
2026 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2027 ip4_is_first_fragment (ip0)))
2029 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2031 tcp0 = ip4_next_header (ip0);
2032 tcp0->dst = new_dst_port0;
2033 sum0 = tcp0->checksum;
2034 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2035 ip4_header_t, dst_address);
2036 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2037 ip4_header_t /* cheat */, length);
2038 tcp0->checksum = ip_csum_fold(sum0);
2042 udp0->dst_port = new_dst_port0;
2048 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2050 tcp0 = ip4_next_header (ip0);
2051 sum0 = tcp0->checksum;
2052 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2053 ip4_header_t, dst_address);
2054 tcp0->checksum = ip_csum_fold(sum0);
2061 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2062 vlib_node_runtime_t * node,
2063 vlib_frame_t * frame)
2065 u32 n_left_from, *from, *to_next;
2066 snat_in2out_next_t next_index;
2067 u32 pkts_processed = 0;
2068 snat_main_t *sm = &snat_main;
2069 f64 now = vlib_time_now (vm);
2070 u32 thread_index = vm->thread_index;
2071 snat_main_per_thread_data_t *per_thread_data =
2072 &sm->per_thread_data[thread_index];
2073 u32 *fragments_to_drop = 0;
2074 u32 *fragments_to_loopback = 0;
2076 from = vlib_frame_vector_args (frame);
2077 n_left_from = frame->n_vectors;
2078 next_index = node->cached_next_index;
2080 while (n_left_from > 0)
2084 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2086 while (n_left_from > 0 && n_left_to_next > 0)
2088 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2093 nat_reass_ip4_t *reass0;
2094 udp_header_t * udp0;
2095 tcp_header_t * tcp0;
2096 snat_session_key_t key0;
2097 clib_bihash_kv_8_8_t kv0, value0;
2098 snat_session_t * s0 = 0;
2099 u16 old_port0, new_port0;
2102 /* speculatively enqueue b0 to the current next frame */
2108 n_left_to_next -= 1;
2110 b0 = vlib_get_buffer (vm, bi0);
2111 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2113 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2114 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2117 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2119 next0 = SNAT_IN2OUT_NEXT_DROP;
2120 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2124 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2125 udp0 = ip4_next_header (ip0);
2126 tcp0 = (tcp_header_t *) udp0;
2127 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2129 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2134 &fragments_to_drop);
2136 if (PREDICT_FALSE (!reass0))
2138 next0 = SNAT_IN2OUT_NEXT_DROP;
2139 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2140 nat_log_notice ("maximum reassemblies exceeded");
2144 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2146 key0.addr = ip0->src_address;
2147 key0.port = udp0->src_port;
2148 key0.protocol = proto0;
2149 key0.fib_index = rx_fib_index0;
2150 kv0.key = key0.as_u64;
2152 if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2154 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2155 ip0, proto0, rx_fib_index0, thread_index)))
2158 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2159 &s0, node, next0, thread_index);
2161 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2164 reass0->sess_index = s0 - per_thread_data->sessions;
2168 s0 = pool_elt_at_index (per_thread_data->sessions,
2170 reass0->sess_index = value0.value;
2172 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2176 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2178 if (nat_ip4_reass_add_fragment (reass0, bi0))
2180 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2181 nat_log_notice ("maximum fragments per reassembly exceeded");
2182 next0 = SNAT_IN2OUT_NEXT_DROP;
2188 s0 = pool_elt_at_index (per_thread_data->sessions,
2189 reass0->sess_index);
2192 old_addr0 = ip0->src_address.as_u32;
2193 ip0->src_address = s0->out2in.addr;
2194 new_addr0 = ip0->src_address.as_u32;
2195 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2197 sum0 = ip0->checksum;
2198 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2200 src_address /* changed member */);
2201 ip0->checksum = ip_csum_fold (sum0);
2203 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2205 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2207 old_port0 = tcp0->src_port;
2208 tcp0->src_port = s0->out2in.port;
2209 new_port0 = tcp0->src_port;
2211 sum0 = tcp0->checksum;
2212 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2214 dst_address /* changed member */);
2215 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2216 ip4_header_t /* cheat */,
2217 length /* changed member */);
2218 tcp0->checksum = ip_csum_fold(sum0);
2222 old_port0 = udp0->src_port;
2223 udp0->src_port = s0->out2in.port;
2229 nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2230 s0->ext_host_port, proto0);
2233 nat44_session_update_counters (s0, now,
2234 vlib_buffer_length_in_chain (vm, b0));
2235 /* Per-user LRU list maintenance */
2236 nat44_session_update_lru (sm, s0, thread_index);
2239 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2240 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2242 nat44_in2out_reass_trace_t *t =
2243 vlib_add_trace (vm, node, b0, sizeof (*t));
2244 t->cached = cached0;
2245 t->sw_if_index = sw_if_index0;
2246 t->next_index = next0;
2256 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2258 /* verify speculative enqueue, maybe switch current next frame */
2259 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2260 to_next, n_left_to_next,
2264 if (n_left_from == 0 && vec_len (fragments_to_loopback))
2266 from = vlib_frame_vector_args (frame);
2267 u32 len = vec_len (fragments_to_loopback);
2268 if (len <= VLIB_FRAME_SIZE)
2270 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2272 vec_reset_length (fragments_to_loopback);
2277 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2278 sizeof (u32) * VLIB_FRAME_SIZE);
2279 n_left_from = VLIB_FRAME_SIZE;
2280 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2285 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2288 vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2289 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2292 nat_send_all_to_node (vm, fragments_to_drop, node,
2293 &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2294 SNAT_IN2OUT_NEXT_DROP);
2296 vec_free (fragments_to_drop);
2297 vec_free (fragments_to_loopback);
2298 return frame->n_vectors;
2301 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2302 .function = nat44_in2out_reass_node_fn,
2303 .name = "nat44-in2out-reass",
2304 .vector_size = sizeof (u32),
2305 .format_trace = format_nat44_in2out_reass_trace,
2306 .type = VLIB_NODE_TYPE_INTERNAL,
2308 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2309 .error_strings = snat_in2out_error_strings,
2311 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2313 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2314 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2315 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2316 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2317 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2321 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2322 nat44_in2out_reass_node_fn);
2324 /*******************************/
2325 /*** endpoint-dependent mode ***/
2326 /*******************************/
2328 static_always_inline int
2329 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
2331 icmp46_header_t *icmp0;
2332 nat_ed_ses_key_t key0;
2333 icmp_echo_header_t *echo0, *inner_echo0 = 0;
2334 ip4_header_t *inner_ip0 = 0;
2335 void *l4_header = 0;
2336 icmp46_header_t *inner_icmp0;
2338 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2339 echo0 = (icmp_echo_header_t *)(icmp0+1);
2341 if (!icmp_is_error_message (icmp0))
2343 key0.proto = IP_PROTOCOL_ICMP;
2344 key0.l_addr = ip0->src_address;
2345 key0.r_addr = ip0->dst_address;
2346 key0.l_port = echo0->identifier;
2351 inner_ip0 = (ip4_header_t *)(echo0+1);
2352 l4_header = ip4_next_header (inner_ip0);
2353 key0.proto = inner_ip0->protocol;
2354 key0.r_addr = inner_ip0->src_address;
2355 key0.l_addr = inner_ip0->dst_address;
2356 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
2358 case SNAT_PROTOCOL_ICMP:
2359 inner_icmp0 = (icmp46_header_t*)l4_header;
2360 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2362 key0.l_port = inner_echo0->identifier;
2364 case SNAT_PROTOCOL_UDP:
2365 case SNAT_PROTOCOL_TCP:
2366 key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2367 key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
2370 return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
2378 slow_path_ed (snat_main_t *sm,
2381 clib_bihash_kv_16_8_t *kv,
2382 snat_session_t ** sessionp,
2383 vlib_node_runtime_t * node,
2389 snat_session_key_t key0, key1;
2390 u8 lb = 0, is_sm = 0;
2391 u32 address_index = ~0;
2392 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2393 nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
2394 u32 proto = ip_proto_to_snat_proto (key->proto);
2395 nat_outside_fib_t *outside_fib;
2396 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2397 fib_prefix_t pfx = {
2398 .fp_proto = FIB_PROTOCOL_IP4,
2401 .ip4.as_u32 = key->r_addr.as_u32,
2405 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
2407 b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2408 nat_ipfix_logging_max_sessions(sm->max_translations);
2409 nat_log_notice ("maximum sessions exceeded");
2410 return SNAT_IN2OUT_NEXT_DROP;
2413 key0.addr = key->l_addr;
2414 key0.port = key->l_port;
2415 key1.protocol = key0.protocol = proto;
2416 key0.fib_index = rx_fib_index;
2417 key1.fib_index = sm->outside_fib_index;
2418 /* First try to match static mapping by local address and port */
2419 if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb))
2421 /* Try to create dynamic translation */
2422 if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
2423 thread_index, &key1,
2425 sm->port_per_thread,
2426 tsm->snat_thread_index))
2428 nat_log_notice ("addresses exhausted");
2429 b->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2430 return SNAT_IN2OUT_NEXT_DROP;
2436 u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
2439 nat_log_warn ("create NAT user failed");
2440 return SNAT_IN2OUT_NEXT_DROP;
2443 s = nat_session_alloc_or_recycle (sm, u, thread_index);
2446 nat44_delete_user_with_no_session (sm, u, thread_index);
2447 nat_log_warn ("create NAT session failed");
2448 return SNAT_IN2OUT_NEXT_DROP;
2451 user_session_increment (sm, u, is_sm);
2453 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2455 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
2456 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2457 s->outside_address_index = address_index;
2458 s->ext_host_addr = key->r_addr;
2459 s->ext_host_port = key->r_port;
2462 s->out2in.protocol = key0.protocol;
2464 switch (vec_len (sm->outside_fibs))
2467 s->out2in.fib_index = sm->outside_fib_index;
2470 s->out2in.fib_index = sm->outside_fibs[0].fib_index;
2473 vec_foreach (outside_fib, sm->outside_fibs)
2475 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2476 if (FIB_NODE_INDEX_INVALID != fei)
2478 if (fib_entry_get_resolving_interface (fei) != ~0)
2480 s->out2in.fib_index = outside_fib->fib_index;
2488 /* Add to lookup tables */
2489 kv->value = s - tsm->sessions;
2490 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, kv, 1))
2491 nat_log_notice ("in2out-ed key add failed");
2493 make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
2494 key1.port, key->r_port);
2495 kv->value = s - tsm->sessions;
2496 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, kv, 1))
2497 nat_log_notice ("out2in-ed key add failed");
2502 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
2503 s->out2in.addr.as_u32,
2507 s->in2out.fib_index);
2511 static_always_inline int
2512 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
2513 u32 sw_if_index, ip4_header_t * ip, u32 proto,
2514 u32 rx_fib_index, u32 thread_index)
2516 udp_header_t *udp = ip4_next_header (ip);
2517 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2518 clib_bihash_kv_16_8_t kv, value;
2519 snat_session_key_t key0, key1;
2521 make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
2522 sm->outside_fib_index, udp->dst_port, udp->src_port);
2524 /* NAT packet aimed at external address if */
2525 /* has active sessions */
2526 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2528 key0.addr = ip->dst_address;
2529 key0.port = udp->dst_port;
2530 key0.protocol = proto;
2531 key0.fib_index = sm->outside_fib_index;
2532 /* or is static mappings */
2533 if (!snat_static_mapping_match(sm, key0, &key1, 1, 0, 0, 0))
2539 if (sm->forwarding_enabled)
2542 return snat_not_translate_fast(sm, node, sw_if_index, ip, proto, rx_fib_index);
2545 static_always_inline int
2546 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
2547 u32 thread_index, f64 now,
2548 vlib_main_t * vm, vlib_buffer_t * b)
2550 nat_ed_ses_key_t key;
2551 clib_bihash_kv_16_8_t kv, value;
2553 snat_session_t *s = 0;
2554 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2556 if (!sm->forwarding_enabled)
2559 if (ip->protocol == IP_PROTOCOL_ICMP)
2561 key.as_u64[0] = key.as_u64[1] = 0;
2562 if (icmp_get_ed_key (ip, &key))
2565 kv.key[0] = key.as_u64[0];
2566 kv.key[1] = key.as_u64[1];
2568 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
2570 udp = ip4_next_header(ip);
2571 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
2572 udp->src_port, udp->dst_port);
2576 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
2580 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2582 s = pool_elt_at_index (tsm->sessions, value.value);
2583 if (is_fwd_bypass_session (s))
2585 if (ip->protocol == IP_PROTOCOL_TCP)
2587 tcp_header_t *tcp = ip4_next_header(ip);
2588 if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
2591 /* Per-user LRU list maintenance */
2592 nat44_session_update_lru (sm, s, thread_index);
2594 nat44_session_update_counters (s, now,
2595 vlib_buffer_length_in_chain (vm, b));
2605 static_always_inline int
2606 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
2607 u8 proto, u16 src_port, u16 dst_port,
2608 u32 thread_index, u32 sw_if_index)
2610 clib_bihash_kv_16_8_t kv, value;
2611 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2612 snat_interface_t *i;
2614 u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2617 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, fib_index,
2618 src_port, dst_port);
2619 if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2623 make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, fib_index,
2624 dst_port, src_port);
2625 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2627 s = pool_elt_at_index (tsm->sessions, value.value);
2628 if (is_fwd_bypass_session (s))
2632 pool_foreach (i, sm->output_feature_interfaces,
2634 if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
2644 icmp_match_in2out_ed(snat_main_t *sm, vlib_node_runtime_t *node,
2645 u32 thread_index, vlib_buffer_t *b, ip4_header_t *ip,
2646 u8 *p_proto, snat_session_key_t *p_value,
2647 u8 *p_dont_translate, void *d, void *e)
2649 icmp46_header_t *icmp;
2652 nat_ed_ses_key_t key;
2653 snat_session_t *s = 0;
2654 u8 dont_translate = 0;
2655 clib_bihash_kv_16_8_t kv, value;
2658 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2660 icmp = (icmp46_header_t *) ip4_next_header (ip);
2661 sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
2662 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2664 key.as_u64[0] = key.as_u64[1] = 0;
2665 err = icmp_get_ed_key (ip, &key);
2668 b->error = node->errors[err];
2669 next = SNAT_IN2OUT_NEXT_DROP;
2672 key.fib_index = rx_fib_index;
2674 kv.key[0] = key.as_u64[0];
2675 kv.key[1] = key.as_u64[1];
2677 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2679 if (vnet_buffer(b)->sw_if_index[VLIB_TX] != ~0)
2681 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(sm, ip,
2682 key.proto, key.l_port, key.r_port, thread_index, sw_if_index)))
2690 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node, sw_if_index,
2691 ip, SNAT_PROTOCOL_ICMP, rx_fib_index, thread_index)))
2698 if (PREDICT_FALSE(icmp_is_error_message (icmp)))
2700 b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2701 next = SNAT_IN2OUT_NEXT_DROP;
2705 next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
2708 if (PREDICT_FALSE (next == SNAT_IN2OUT_NEXT_DROP))
2713 if (PREDICT_FALSE(icmp->type != ICMP4_echo_request &&
2714 icmp->type != ICMP4_echo_reply &&
2715 !icmp_is_error_message (icmp)))
2717 b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2718 next = SNAT_IN2OUT_NEXT_DROP;
2722 s = pool_elt_at_index (tsm->sessions, value.value);
2725 *p_proto = ip_proto_to_snat_proto (key.proto);
2728 *p_value = s->out2in;
2729 *p_dont_translate = dont_translate;
2731 *(snat_session_t**)d = s;
2736 nat44_ed_hairpinning_unknown_proto (snat_main_t *sm,
2740 u32 old_addr, new_addr = 0, ti = 0;
2741 clib_bihash_kv_8_8_t kv, value;
2742 clib_bihash_kv_16_8_t s_kv, s_value;
2743 snat_static_mapping_t *m;
2746 snat_main_per_thread_data_t *tsm;
2748 if (sm->num_workers > 1)
2749 ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
2751 ti = sm->num_workers;
2752 tsm = &sm->per_thread_data[ti];
2754 old_addr = ip->dst_address.as_u32;
2755 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2756 sm->outside_fib_index, 0, 0);
2757 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2759 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
2760 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2763 m = pool_elt_at_index (sm->static_mappings, value.value);
2764 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2765 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
2766 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
2770 s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
2771 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2772 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2773 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
2776 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2777 ip->checksum = ip_csum_fold (sum);
2780 static snat_session_t *
2781 nat44_ed_in2out_unknown_proto (snat_main_t *sm,
2788 vlib_node_runtime_t * node)
2790 clib_bihash_kv_8_8_t kv, value;
2791 clib_bihash_kv_16_8_t s_kv, s_value;
2792 snat_static_mapping_t *m;
2793 u32 old_addr, new_addr = 0;
2796 dlist_elt_t *head, *elt;
2797 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2798 u32 elt_index, head_index, ses_index;
2800 u32 address_index = ~0, outside_fib_index = sm->outside_fib_index;
2803 nat_outside_fib_t *outside_fib;
2804 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2805 fib_prefix_t pfx = {
2806 .fp_proto = FIB_PROTOCOL_IP4,
2809 .ip4.as_u32 = ip->dst_address.as_u32,
2813 switch (vec_len (sm->outside_fibs))
2816 outside_fib_index = sm->outside_fib_index;
2819 outside_fib_index = sm->outside_fibs[0].fib_index;
2822 vec_foreach (outside_fib, sm->outside_fibs)
2824 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2825 if (FIB_NODE_INDEX_INVALID != fei)
2827 if (fib_entry_get_resolving_interface (fei) != ~0)
2829 outside_fib_index = outside_fib->fib_index;
2836 old_addr = ip->src_address.as_u32;
2838 make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
2839 rx_fib_index, 0, 0);
2841 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
2843 s = pool_elt_at_index (tsm->sessions, s_value.value);
2844 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
2848 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
2850 b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2851 nat_ipfix_logging_max_sessions(sm->max_translations);
2852 nat_log_notice ("maximum sessions exceeded");
2856 u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
2860 nat_log_warn ("create NAT user failed");
2864 make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
2866 /* Try to find static mapping first */
2867 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
2869 m = pool_elt_at_index (sm->static_mappings, value.value);
2870 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
2874 /* Fallback to 3-tuple key */
2877 /* Choose same out address as for TCP/UDP session to same destination */
2878 head_index = u->sessions_per_user_list_head_index;
2879 head = pool_elt_at_index (tsm->list_pool, head_index);
2880 elt_index = head->next;
2881 if (PREDICT_FALSE (elt_index == ~0))
2885 elt = pool_elt_at_index (tsm->list_pool, elt_index);
2886 ses_index = elt->value;
2889 while (ses_index != ~0)
2891 s = pool_elt_at_index (tsm->sessions, ses_index);
2892 elt_index = elt->next;
2893 elt = pool_elt_at_index (tsm->list_pool, elt_index);
2894 ses_index = elt->value;
2896 if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
2898 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
2899 address_index = s->outside_address_index;
2901 make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
2902 ip->protocol, outside_fib_index, 0, 0);
2903 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2910 for (i = 0; i < vec_len (sm->addresses); i++)
2912 make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
2913 ip->protocol, outside_fib_index, 0, 0);
2914 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2916 new_addr = ip->src_address.as_u32 =
2917 sm->addresses[i].addr.as_u32;
2926 s = nat_session_alloc_or_recycle (sm, u, thread_index);
2929 nat44_delete_user_with_no_session (sm, u, thread_index);
2930 nat_log_warn ("create NAT session failed");
2934 s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
2935 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
2936 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2937 s->outside_address_index = address_index;
2938 s->out2in.addr.as_u32 = new_addr;
2939 s->out2in.fib_index = outside_fib_index;
2940 s->in2out.addr.as_u32 = old_addr;
2941 s->in2out.fib_index = rx_fib_index;
2942 s->in2out.port = s->out2in.port = ip->protocol;
2944 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2945 user_session_increment (sm, u, is_sm);
2947 /* Add to lookup tables */
2948 make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
2949 rx_fib_index, 0, 0);
2950 s_kv.value = s - tsm->sessions;
2951 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2952 nat_log_notice ("in2out key add failed");
2954 make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
2955 outside_fib_index, 0, 0);
2956 s_kv.value = s - tsm->sessions;
2957 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2958 nat_log_notice ("out2in key add failed");
2961 /* Update IP checksum */
2963 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
2964 ip->checksum = ip_csum_fold (sum);
2967 nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
2968 /* Per-user LRU list maintenance */
2969 nat44_session_update_lru (sm, s, thread_index);
2972 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2973 nat44_ed_hairpinning_unknown_proto(sm, b, ip);
2975 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2976 vnet_buffer(b)->sw_if_index[VLIB_TX] = outside_fib_index;
2982 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
2983 vlib_node_runtime_t * node,
2984 vlib_frame_t * frame, int is_slow_path,
2985 int is_output_feature)
2987 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2988 snat_in2out_next_t next_index;
2989 snat_main_t *sm = &snat_main;
2990 f64 now = vlib_time_now (vm);
2991 u32 thread_index = vm->thread_index;
2992 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2994 stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
2995 nat44_ed_in2out_node.index;
2997 from = vlib_frame_vector_args (frame);
2998 n_left_from = frame->n_vectors;
2999 next_index = node->cached_next_index;
3001 while (n_left_from > 0)
3005 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3007 while (n_left_from >= 4 && n_left_to_next >= 2)
3010 vlib_buffer_t *b0, *b1;
3011 u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3012 new_addr0, old_addr0;
3013 u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
3014 new_addr1, old_addr1;
3015 u16 old_port0, new_port0, old_port1, new_port1;
3016 ip4_header_t *ip0, *ip1;
3017 udp_header_t *udp0, *udp1;
3018 tcp_header_t *tcp0, *tcp1;
3019 icmp46_header_t *icmp0, *icmp1;
3020 snat_session_t *s0 = 0, *s1 = 0;
3021 clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
3022 ip_csum_t sum0, sum1;
3024 /* Prefetch next iteration. */
3026 vlib_buffer_t * p2, * p3;
3028 p2 = vlib_get_buffer (vm, from[2]);
3029 p3 = vlib_get_buffer (vm, from[3]);
3031 vlib_prefetch_buffer_header (p2, LOAD);
3032 vlib_prefetch_buffer_header (p3, LOAD);
3034 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3035 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3038 /* speculatively enqueue b0 and b1 to the current next frame */
3039 to_next[0] = bi0 = from[0];
3040 to_next[1] = bi1 = from[1];
3044 n_left_to_next -= 2;
3046 b0 = vlib_get_buffer (vm, bi0);
3047 b1 = vlib_get_buffer (vm, bi1);
3049 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3051 if (is_output_feature)
3052 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3054 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3057 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3058 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3061 if (PREDICT_FALSE(ip0->ttl == 1))
3063 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3064 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3065 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3067 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3071 udp0 = ip4_next_header (ip0);
3072 tcp0 = (tcp_header_t *) udp0;
3073 icmp0 = (icmp46_header_t *) udp0;
3074 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3078 if (PREDICT_FALSE (proto0 == ~0))
3080 s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3082 thread_index, now, vm,
3085 next0 = SNAT_IN2OUT_NEXT_DROP;
3089 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3091 next0 = icmp_in2out_slow_path
3092 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3093 next0, now, thread_index, &s0);
3099 if (is_output_feature)
3101 if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3102 sm, ip0, thread_index, now, vm, b0)))
3106 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3108 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3112 if (ip4_is_fragment (ip0))
3114 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3115 next0 = SNAT_IN2OUT_NEXT_DROP;
3120 make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3121 rx_fib_index0, udp0->src_port, udp0->dst_port);
3123 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3127 if (is_output_feature)
3129 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3130 sm, ip0, ip0->protocol, udp0->src_port,
3131 udp0->dst_port, thread_index, sw_if_index0)))
3136 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3137 sw_if_index0, ip0, proto0, rx_fib_index0,
3142 next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3143 next0, thread_index);
3145 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3150 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3156 s0 = pool_elt_at_index (tsm->sessions, value0.value);
3159 b0->flags |= VNET_BUFFER_F_IS_NATED;
3161 if (!is_output_feature)
3162 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3164 old_addr0 = ip0->src_address.as_u32;
3165 new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3166 sum0 = ip0->checksum;
3167 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3169 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3170 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3171 s0->ext_host_addr.as_u32, ip4_header_t,
3173 ip0->checksum = ip_csum_fold (sum0);
3175 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3177 old_port0 = tcp0->src_port;
3178 new_port0 = tcp0->src_port = s0->out2in.port;
3180 sum0 = tcp0->checksum;
3181 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3183 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3185 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3187 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3188 s0->ext_host_addr.as_u32,
3189 ip4_header_t, dst_address);
3190 sum0 = ip_csum_update (sum0, tcp0->dst_port,
3191 s0->ext_host_port, ip4_header_t,
3193 tcp0->dst_port = s0->ext_host_port;
3194 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3196 tcp0->checksum = ip_csum_fold(sum0);
3197 if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3202 udp0->src_port = s0->out2in.port;
3204 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3206 udp0->dst_port = s0->ext_host_port;
3207 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3212 nat44_session_update_counters (s0, now,
3213 vlib_buffer_length_in_chain (vm, b0));
3214 /* Per-user LRU list maintenance */
3215 nat44_session_update_lru (sm, s0, thread_index);
3218 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3219 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3221 snat_in2out_trace_t *t =
3222 vlib_add_trace (vm, node, b0, sizeof (*t));
3223 t->is_slow_path = is_slow_path;
3224 t->sw_if_index = sw_if_index0;
3225 t->next_index = next0;
3226 t->session_index = ~0;
3228 t->session_index = s0 - tsm->sessions;
3231 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3234 next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3236 if (is_output_feature)
3237 iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
3239 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
3242 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3243 rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3246 if (PREDICT_FALSE(ip1->ttl == 1))
3248 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3249 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3250 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3252 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3256 udp1 = ip4_next_header (ip1);
3257 tcp1 = (tcp_header_t *) udp1;
3258 icmp1 = (icmp46_header_t *) udp1;
3259 proto1 = ip_proto_to_snat_proto (ip1->protocol);
3263 if (PREDICT_FALSE (proto1 == ~0))
3265 s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
3267 thread_index, now, vm,
3270 next1 = SNAT_IN2OUT_NEXT_DROP;
3274 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
3276 next1 = icmp_in2out_slow_path
3277 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
3278 next1, now, thread_index, &s1);
3284 if (is_output_feature)
3286 if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3287 sm, ip1, thread_index, now, vm, b1)))
3291 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
3293 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3297 if (ip4_is_fragment (ip1))
3299 b1->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3300 next1 = SNAT_IN2OUT_NEXT_DROP;
3305 make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address, ip1->protocol,
3306 rx_fib_index1, udp1->src_port, udp1->dst_port);
3308 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
3312 if (is_output_feature)
3314 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3315 sm, ip1, ip1->protocol, udp1->src_port,
3316 udp1->dst_port, thread_index, sw_if_index1)))
3321 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3322 sw_if_index1, ip1, proto1, rx_fib_index1,
3327 next1 = slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
3328 next1, thread_index);
3330 if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
3335 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3341 s1 = pool_elt_at_index (tsm->sessions, value1.value);
3344 b1->flags |= VNET_BUFFER_F_IS_NATED;
3346 if (!is_output_feature)
3347 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
3349 old_addr1 = ip1->src_address.as_u32;
3350 new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
3351 sum1 = ip1->checksum;
3352 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3354 if (PREDICT_FALSE (is_twice_nat_session (s1)))
3355 sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3356 s1->ext_host_addr.as_u32, ip4_header_t,
3358 ip1->checksum = ip_csum_fold (sum1);
3360 if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
3362 old_port1 = tcp1->src_port;
3363 new_port1 = tcp1->src_port = s1->out2in.port;
3365 sum1 = tcp1->checksum;
3366 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3368 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
3370 if (PREDICT_FALSE (is_twice_nat_session (s1)))
3372 sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3373 s1->ext_host_addr.as_u32,
3374 ip4_header_t, dst_address);
3375 sum1 = ip_csum_update (sum1, tcp1->dst_port,
3376 s1->ext_host_port, ip4_header_t,
3378 tcp1->dst_port = s1->ext_host_port;
3379 ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3381 tcp1->checksum = ip_csum_fold(sum1);
3382 if (nat44_set_tcp_session_state_i2o (sm, s1, tcp1, thread_index))
3387 udp1->src_port = s1->out2in.port;
3389 if (PREDICT_FALSE (is_twice_nat_session (s1)))
3391 udp1->dst_port = s1->ext_host_port;
3392 ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3397 nat44_session_update_counters (s1, now,
3398 vlib_buffer_length_in_chain (vm, b1));
3399 /* Per-user LRU list maintenance */
3400 nat44_session_update_lru (sm, s1, thread_index);
3403 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3404 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3406 snat_in2out_trace_t *t =
3407 vlib_add_trace (vm, node, b1, sizeof (*t));
3408 t->is_slow_path = is_slow_path;
3409 t->sw_if_index = sw_if_index1;
3410 t->next_index = next1;
3411 t->session_index = ~0;
3413 t->session_index = s1 - tsm->sessions;
3416 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3418 /* verify speculative enqueues, maybe switch current next frame */
3419 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3420 to_next, n_left_to_next,
3421 bi0, bi1, next0, next1);
3424 while (n_left_from > 0 && n_left_to_next > 0)
3428 u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3429 new_addr0, old_addr0;
3430 u16 old_port0, new_port0;
3434 icmp46_header_t * icmp0;
3435 snat_session_t *s0 = 0;
3436 clib_bihash_kv_16_8_t kv0, value0;
3439 /* speculatively enqueue b0 to the current next frame */
3445 n_left_to_next -= 1;
3447 b0 = vlib_get_buffer (vm, bi0);
3448 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3450 if (is_output_feature)
3451 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3453 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3456 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3457 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3460 if (PREDICT_FALSE(ip0->ttl == 1))
3462 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3463 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3464 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3466 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3470 udp0 = ip4_next_header (ip0);
3471 tcp0 = (tcp_header_t *) udp0;
3472 icmp0 = (icmp46_header_t *) udp0;
3473 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3477 if (PREDICT_FALSE (proto0 == ~0))
3479 s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3481 thread_index, now, vm,
3484 next0 = SNAT_IN2OUT_NEXT_DROP;
3488 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3490 next0 = icmp_in2out_slow_path
3491 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3492 next0, now, thread_index, &s0);
3498 if (is_output_feature)
3500 if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3501 sm, ip0, thread_index, now, vm, b0)))
3505 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3507 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3511 if (ip4_is_fragment (ip0))
3513 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3514 next0 = SNAT_IN2OUT_NEXT_DROP;
3519 make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3520 rx_fib_index0, udp0->src_port, udp0->dst_port);
3522 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3526 if (is_output_feature)
3528 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3529 sm, ip0, ip0->protocol, udp0->src_port,
3530 udp0->dst_port, thread_index, sw_if_index0)))
3535 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3536 sw_if_index0, ip0, proto0, rx_fib_index0,
3541 next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3542 next0, thread_index);
3544 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3549 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3555 s0 = pool_elt_at_index (tsm->sessions, value0.value);
3558 b0->flags |= VNET_BUFFER_F_IS_NATED;
3560 if (!is_output_feature)
3561 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3563 old_addr0 = ip0->src_address.as_u32;
3564 new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3565 sum0 = ip0->checksum;
3566 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3568 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3569 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3570 s0->ext_host_addr.as_u32, ip4_header_t,
3572 ip0->checksum = ip_csum_fold (sum0);
3574 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3576 old_port0 = tcp0->src_port;
3577 new_port0 = tcp0->src_port = s0->out2in.port;
3579 sum0 = tcp0->checksum;
3580 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3582 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3584 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3586 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3587 s0->ext_host_addr.as_u32,
3588 ip4_header_t, dst_address);
3589 sum0 = ip_csum_update (sum0, tcp0->dst_port,
3590 s0->ext_host_port, ip4_header_t,
3592 tcp0->dst_port = s0->ext_host_port;
3593 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3595 tcp0->checksum = ip_csum_fold(sum0);
3596 if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3601 udp0->src_port = s0->out2in.port;
3603 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3605 udp0->dst_port = s0->ext_host_port;
3606 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3611 nat44_session_update_counters (s0, now,
3612 vlib_buffer_length_in_chain (vm, b0));
3613 /* Per-user LRU list maintenance */
3614 nat44_session_update_lru (sm, s0, thread_index);
3617 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3618 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3620 snat_in2out_trace_t *t =
3621 vlib_add_trace (vm, node, b0, sizeof (*t));
3622 t->is_slow_path = is_slow_path;
3623 t->sw_if_index = sw_if_index0;
3624 t->next_index = next0;
3625 t->session_index = ~0;
3627 t->session_index = s0 - tsm->sessions;
3630 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3632 /* verify speculative enqueue, maybe switch current next frame */
3633 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3634 to_next, n_left_to_next,
3638 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3641 vlib_node_increment_counter (vm, stats_node_index,
3642 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3644 return frame->n_vectors;
3648 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
3649 vlib_node_runtime_t * node,
3650 vlib_frame_t * frame)
3652 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
3655 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
3656 .function = nat44_ed_in2out_fast_path_fn,
3657 .name = "nat44-ed-in2out",
3658 .vector_size = sizeof (u32),
3659 .format_trace = format_snat_in2out_trace,
3660 .type = VLIB_NODE_TYPE_INTERNAL,
3662 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3663 .error_strings = snat_in2out_error_strings,
3665 .runtime_data_bytes = sizeof (snat_runtime_t),
3667 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3669 /* edit / add dispositions here */
3671 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3672 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3673 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3674 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3675 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3679 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node, nat44_ed_in2out_fast_path_fn);
3682 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
3683 vlib_node_runtime_t * node,
3684 vlib_frame_t * frame)
3686 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
3689 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
3690 .function = nat44_ed_in2out_output_fast_path_fn,
3691 .name = "nat44-ed-in2out-output",
3692 .vector_size = sizeof (u32),
3693 .format_trace = format_snat_in2out_trace,
3694 .type = VLIB_NODE_TYPE_INTERNAL,
3696 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3697 .error_strings = snat_in2out_error_strings,
3699 .runtime_data_bytes = sizeof (snat_runtime_t),
3701 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3703 /* edit / add dispositions here */
3705 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3706 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3707 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3708 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3709 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3713 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
3714 nat44_ed_in2out_output_fast_path_fn);
3717 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
3718 vlib_node_runtime_t * node,
3719 vlib_frame_t * frame)
3721 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
3724 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
3725 .function = nat44_ed_in2out_slow_path_fn,
3726 .name = "nat44-ed-in2out-slowpath",
3727 .vector_size = sizeof (u32),
3728 .format_trace = format_snat_in2out_trace,
3729 .type = VLIB_NODE_TYPE_INTERNAL,
3731 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3732 .error_strings = snat_in2out_error_strings,
3734 .runtime_data_bytes = sizeof (snat_runtime_t),
3736 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3738 /* edit / add dispositions here */
3740 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3741 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3742 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3743 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3744 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3748 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
3749 nat44_ed_in2out_slow_path_fn);
3752 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
3753 vlib_node_runtime_t * node,
3754 vlib_frame_t * frame)
3756 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
3759 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
3760 .function = nat44_ed_in2out_output_slow_path_fn,
3761 .name = "nat44-ed-in2out-output-slowpath",
3762 .vector_size = sizeof (u32),
3763 .format_trace = format_snat_in2out_trace,
3764 .type = VLIB_NODE_TYPE_INTERNAL,
3766 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3767 .error_strings = snat_in2out_error_strings,
3769 .runtime_data_bytes = sizeof (snat_runtime_t),
3771 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3773 /* edit / add dispositions here */
3775 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3776 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3777 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3778 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3779 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3783 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
3784 nat44_ed_in2out_output_slow_path_fn);
3786 /**************************/
3787 /*** deterministic mode ***/
3788 /**************************/
3790 snat_det_in2out_node_fn (vlib_main_t * vm,
3791 vlib_node_runtime_t * node,
3792 vlib_frame_t * frame)
3794 u32 n_left_from, * from, * to_next;
3795 snat_in2out_next_t next_index;
3796 u32 pkts_processed = 0;
3797 snat_main_t * sm = &snat_main;
3798 u32 now = (u32) vlib_time_now (vm);
3799 u32 thread_index = vm->thread_index;
3801 from = vlib_frame_vector_args (frame);
3802 n_left_from = frame->n_vectors;
3803 next_index = node->cached_next_index;
3805 while (n_left_from > 0)
3809 vlib_get_next_frame (vm, node, next_index,
3810 to_next, n_left_to_next);
3812 while (n_left_from >= 4 && n_left_to_next >= 2)
3815 vlib_buffer_t * b0, * b1;
3817 u32 sw_if_index0, sw_if_index1;
3818 ip4_header_t * ip0, * ip1;
3819 ip_csum_t sum0, sum1;
3820 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
3821 u16 old_port0, new_port0, lo_port0, i0;
3822 u16 old_port1, new_port1, lo_port1, i1;
3823 udp_header_t * udp0, * udp1;
3824 tcp_header_t * tcp0, * tcp1;
3826 snat_det_out_key_t key0, key1;
3827 snat_det_map_t * dm0, * dm1;
3828 snat_det_session_t * ses0 = 0, * ses1 = 0;
3829 u32 rx_fib_index0, rx_fib_index1;
3830 icmp46_header_t * icmp0, * icmp1;
3832 /* Prefetch next iteration. */
3834 vlib_buffer_t * p2, * p3;
3836 p2 = vlib_get_buffer (vm, from[2]);
3837 p3 = vlib_get_buffer (vm, from[3]);
3839 vlib_prefetch_buffer_header (p2, LOAD);
3840 vlib_prefetch_buffer_header (p3, LOAD);
3842 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3843 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3846 /* speculatively enqueue b0 and b1 to the current next frame */
3847 to_next[0] = bi0 = from[0];
3848 to_next[1] = bi1 = from[1];
3852 n_left_to_next -= 2;
3854 b0 = vlib_get_buffer (vm, bi0);
3855 b1 = vlib_get_buffer (vm, bi1);
3857 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3858 next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3860 ip0 = vlib_buffer_get_current (b0);
3861 udp0 = ip4_next_header (ip0);
3862 tcp0 = (tcp_header_t *) udp0;
3864 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3866 if (PREDICT_FALSE(ip0->ttl == 1))
3868 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3869 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3870 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3872 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3876 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3878 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3880 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3881 icmp0 = (icmp46_header_t *) udp0;
3883 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3884 rx_fib_index0, node, next0, thread_index,
3889 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3890 if (PREDICT_FALSE(!dm0))
3892 nat_log_info ("no match for internal host %U",
3893 format_ip4_address, &ip0->src_address);
3894 next0 = SNAT_IN2OUT_NEXT_DROP;
3895 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3899 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3901 key0.ext_host_addr = ip0->dst_address;
3902 key0.ext_host_port = tcp0->dst;
3904 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3905 if (PREDICT_FALSE(!ses0))
3907 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3909 key0.out_port = clib_host_to_net_u16 (lo_port0 +
3910 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3912 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3915 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3918 if (PREDICT_FALSE(!ses0))
3920 /* too many sessions for user, send ICMP error packet */
3922 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3923 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3924 ICMP4_destination_unreachable_destination_unreachable_host,
3926 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3931 new_port0 = ses0->out.out_port;
3933 old_addr0.as_u32 = ip0->src_address.as_u32;
3934 ip0->src_address.as_u32 = new_addr0.as_u32;
3935 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3937 sum0 = ip0->checksum;
3938 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3940 src_address /* changed member */);
3941 ip0->checksum = ip_csum_fold (sum0);
3943 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3945 if (tcp0->flags & TCP_FLAG_SYN)
3946 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3947 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3948 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3949 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3950 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3951 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3952 snat_det_ses_close(dm0, ses0);
3953 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3954 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3955 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3956 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3958 old_port0 = tcp0->src;
3959 tcp0->src = new_port0;
3961 sum0 = tcp0->checksum;
3962 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3964 dst_address /* changed member */);
3965 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3966 ip4_header_t /* cheat */,
3967 length /* changed member */);
3968 tcp0->checksum = ip_csum_fold(sum0);
3972 ses0->state = SNAT_SESSION_UDP_ACTIVE;
3973 old_port0 = udp0->src_port;
3974 udp0->src_port = new_port0;
3980 case SNAT_SESSION_UDP_ACTIVE:
3981 ses0->expire = now + sm->udp_timeout;
3983 case SNAT_SESSION_TCP_SYN_SENT:
3984 case SNAT_SESSION_TCP_FIN_WAIT:
3985 case SNAT_SESSION_TCP_CLOSE_WAIT:
3986 case SNAT_SESSION_TCP_LAST_ACK:
3987 ses0->expire = now + sm->tcp_transitory_timeout;
3989 case SNAT_SESSION_TCP_ESTABLISHED:
3990 ses0->expire = now + sm->tcp_established_timeout;
3995 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3996 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3998 snat_in2out_trace_t *t =
3999 vlib_add_trace (vm, node, b0, sizeof (*t));
4000 t->is_slow_path = 0;
4001 t->sw_if_index = sw_if_index0;
4002 t->next_index = next0;
4003 t->session_index = ~0;
4005 t->session_index = ses0 - dm0->sessions;
4008 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4010 ip1 = vlib_buffer_get_current (b1);
4011 udp1 = ip4_next_header (ip1);
4012 tcp1 = (tcp_header_t *) udp1;
4014 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
4016 if (PREDICT_FALSE(ip1->ttl == 1))
4018 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4019 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
4020 ICMP4_time_exceeded_ttl_exceeded_in_transit,
4022 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4026 proto1 = ip_proto_to_snat_proto (ip1->protocol);
4028 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
4030 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
4031 icmp1 = (icmp46_header_t *) udp1;
4033 next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
4034 rx_fib_index1, node, next1, thread_index,
4039 dm1 = snat_det_map_by_user(sm, &ip1->src_address);
4040 if (PREDICT_FALSE(!dm1))
4042 nat_log_info ("no match for internal host %U",
4043 format_ip4_address, &ip0->src_address);
4044 next1 = SNAT_IN2OUT_NEXT_DROP;
4045 b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4049 snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
4051 key1.ext_host_addr = ip1->dst_address;
4052 key1.ext_host_port = tcp1->dst;
4054 ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
4055 if (PREDICT_FALSE(!ses1))
4057 for (i1 = 0; i1 < dm1->ports_per_host; i1++)
4059 key1.out_port = clib_host_to_net_u16 (lo_port1 +
4060 ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
4062 if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
4065 ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
4068 if (PREDICT_FALSE(!ses1))
4070 /* too many sessions for user, send ICMP error packet */
4072 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4073 icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
4074 ICMP4_destination_unreachable_destination_unreachable_host,
4076 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4081 new_port1 = ses1->out.out_port;
4083 old_addr1.as_u32 = ip1->src_address.as_u32;
4084 ip1->src_address.as_u32 = new_addr1.as_u32;
4085 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4087 sum1 = ip1->checksum;
4088 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4090 src_address /* changed member */);
4091 ip1->checksum = ip_csum_fold (sum1);
4093 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
4095 if (tcp1->flags & TCP_FLAG_SYN)
4096 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
4097 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
4098 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4099 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
4100 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
4101 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
4102 snat_det_ses_close(dm1, ses1);
4103 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4104 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
4105 else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
4106 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4108 old_port1 = tcp1->src;
4109 tcp1->src = new_port1;
4111 sum1 = tcp1->checksum;
4112 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4114 dst_address /* changed member */);
4115 sum1 = ip_csum_update (sum1, old_port1, new_port1,
4116 ip4_header_t /* cheat */,
4117 length /* changed member */);
4118 tcp1->checksum = ip_csum_fold(sum1);
4122 ses1->state = SNAT_SESSION_UDP_ACTIVE;
4123 old_port1 = udp1->src_port;
4124 udp1->src_port = new_port1;
4130 case SNAT_SESSION_UDP_ACTIVE:
4131 ses1->expire = now + sm->udp_timeout;
4133 case SNAT_SESSION_TCP_SYN_SENT:
4134 case SNAT_SESSION_TCP_FIN_WAIT:
4135 case SNAT_SESSION_TCP_CLOSE_WAIT:
4136 case SNAT_SESSION_TCP_LAST_ACK:
4137 ses1->expire = now + sm->tcp_transitory_timeout;
4139 case SNAT_SESSION_TCP_ESTABLISHED:
4140 ses1->expire = now + sm->tcp_established_timeout;
4145 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4146 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
4148 snat_in2out_trace_t *t =
4149 vlib_add_trace (vm, node, b1, sizeof (*t));
4150 t->is_slow_path = 0;
4151 t->sw_if_index = sw_if_index1;
4152 t->next_index = next1;
4153 t->session_index = ~0;
4155 t->session_index = ses1 - dm1->sessions;
4158 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
4160 /* verify speculative enqueues, maybe switch current next frame */
4161 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
4162 to_next, n_left_to_next,
4163 bi0, bi1, next0, next1);
4166 while (n_left_from > 0 && n_left_to_next > 0)
4174 ip4_address_t new_addr0, old_addr0;
4175 u16 old_port0, new_port0, lo_port0, i0;
4176 udp_header_t * udp0;
4177 tcp_header_t * tcp0;
4179 snat_det_out_key_t key0;
4180 snat_det_map_t * dm0;
4181 snat_det_session_t * ses0 = 0;
4183 icmp46_header_t * icmp0;
4185 /* speculatively enqueue b0 to the current next frame */
4191 n_left_to_next -= 1;
4193 b0 = vlib_get_buffer (vm, bi0);
4194 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4196 ip0 = vlib_buffer_get_current (b0);
4197 udp0 = ip4_next_header (ip0);
4198 tcp0 = (tcp_header_t *) udp0;
4200 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4202 if (PREDICT_FALSE(ip0->ttl == 1))
4204 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4205 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4206 ICMP4_time_exceeded_ttl_exceeded_in_transit,
4208 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4212 proto0 = ip_proto_to_snat_proto (ip0->protocol);
4214 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4216 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4217 icmp0 = (icmp46_header_t *) udp0;
4219 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4220 rx_fib_index0, node, next0, thread_index,
4225 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4226 if (PREDICT_FALSE(!dm0))
4228 nat_log_info ("no match for internal host %U",
4229 format_ip4_address, &ip0->src_address);
4230 next0 = SNAT_IN2OUT_NEXT_DROP;
4231 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4235 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4237 key0.ext_host_addr = ip0->dst_address;
4238 key0.ext_host_port = tcp0->dst;
4240 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4241 if (PREDICT_FALSE(!ses0))
4243 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4245 key0.out_port = clib_host_to_net_u16 (lo_port0 +
4246 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4248 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4251 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4254 if (PREDICT_FALSE(!ses0))
4256 /* too many sessions for user, send ICMP error packet */
4258 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4259 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4260 ICMP4_destination_unreachable_destination_unreachable_host,
4262 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4267 new_port0 = ses0->out.out_port;
4269 old_addr0.as_u32 = ip0->src_address.as_u32;
4270 ip0->src_address.as_u32 = new_addr0.as_u32;
4271 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4273 sum0 = ip0->checksum;
4274 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4276 src_address /* changed member */);
4277 ip0->checksum = ip_csum_fold (sum0);
4279 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4281 if (tcp0->flags & TCP_FLAG_SYN)
4282 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4283 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4284 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4285 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4286 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4287 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4288 snat_det_ses_close(dm0, ses0);
4289 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4290 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4291 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4292 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4294 old_port0 = tcp0->src;
4295 tcp0->src = new_port0;
4297 sum0 = tcp0->checksum;
4298 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4300 dst_address /* changed member */);
4301 sum0 = ip_csum_update (sum0, old_port0, new_port0,
4302 ip4_header_t /* cheat */,
4303 length /* changed member */);
4304 tcp0->checksum = ip_csum_fold(sum0);
4308 ses0->state = SNAT_SESSION_UDP_ACTIVE;
4309 old_port0 = udp0->src_port;
4310 udp0->src_port = new_port0;
4316 case SNAT_SESSION_UDP_ACTIVE:
4317 ses0->expire = now + sm->udp_timeout;
4319 case SNAT_SESSION_TCP_SYN_SENT:
4320 case SNAT_SESSION_TCP_FIN_WAIT:
4321 case SNAT_SESSION_TCP_CLOSE_WAIT:
4322 case SNAT_SESSION_TCP_LAST_ACK:
4323 ses0->expire = now + sm->tcp_transitory_timeout;
4325 case SNAT_SESSION_TCP_ESTABLISHED:
4326 ses0->expire = now + sm->tcp_established_timeout;
4331 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4332 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4334 snat_in2out_trace_t *t =
4335 vlib_add_trace (vm, node, b0, sizeof (*t));
4336 t->is_slow_path = 0;
4337 t->sw_if_index = sw_if_index0;
4338 t->next_index = next0;
4339 t->session_index = ~0;
4341 t->session_index = ses0 - dm0->sessions;
4344 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4346 /* verify speculative enqueue, maybe switch current next frame */
4347 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4348 to_next, n_left_to_next,
4352 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4355 vlib_node_increment_counter (vm, snat_det_in2out_node.index,
4356 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4358 return frame->n_vectors;
4361 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
4362 .function = snat_det_in2out_node_fn,
4363 .name = "nat44-det-in2out",
4364 .vector_size = sizeof (u32),
4365 .format_trace = format_snat_in2out_trace,
4366 .type = VLIB_NODE_TYPE_INTERNAL,
4368 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4369 .error_strings = snat_in2out_error_strings,
4371 .runtime_data_bytes = sizeof (snat_runtime_t),
4375 /* edit / add dispositions here */
4377 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4378 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4379 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4383 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
4386 * Get address and port values to be used for ICMP packet translation
4387 * and create session if needed
4389 * @param[in,out] sm NAT main
4390 * @param[in,out] node NAT node runtime
4391 * @param[in] thread_index thread index
4392 * @param[in,out] b0 buffer containing packet to be translated
4393 * @param[out] p_proto protocol used for matching
4394 * @param[out] p_value address and port after NAT translation
4395 * @param[out] p_dont_translate if packet should not be translated
4396 * @param d optional parameter
4397 * @param e optional parameter
4399 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
4400 u32 thread_index, vlib_buffer_t *b0,
4401 ip4_header_t *ip0, u8 *p_proto,
4402 snat_session_key_t *p_value,
4403 u8 *p_dont_translate, void *d, void *e)
4405 icmp46_header_t *icmp0;
4409 snat_det_out_key_t key0;
4410 u8 dont_translate = 0;
4412 icmp_echo_header_t *echo0, *inner_echo0 = 0;
4413 ip4_header_t *inner_ip0;
4414 void *l4_header = 0;
4415 icmp46_header_t *inner_icmp0;
4416 snat_det_map_t * dm0 = 0;
4417 ip4_address_t new_addr0;
4419 snat_det_session_t * ses0 = 0;
4420 ip4_address_t in_addr;
4423 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
4424 echo0 = (icmp_echo_header_t *)(icmp0+1);
4425 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4426 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
4428 if (!icmp_is_error_message (icmp0))
4430 protocol = SNAT_PROTOCOL_ICMP;
4431 in_addr = ip0->src_address;
4432 in_port = echo0->identifier;
4436 inner_ip0 = (ip4_header_t *)(echo0+1);
4437 l4_header = ip4_next_header (inner_ip0);
4438 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
4439 in_addr = inner_ip0->dst_address;
4442 case SNAT_PROTOCOL_ICMP:
4443 inner_icmp0 = (icmp46_header_t*)l4_header;
4444 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
4445 in_port = inner_echo0->identifier;
4447 case SNAT_PROTOCOL_UDP:
4448 case SNAT_PROTOCOL_TCP:
4449 in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
4452 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
4453 next0 = SNAT_IN2OUT_NEXT_DROP;
4458 dm0 = snat_det_map_by_user(sm, &in_addr);
4459 if (PREDICT_FALSE(!dm0))
4461 nat_log_info ("no match for internal host %U",
4462 format_ip4_address, &in_addr);
4463 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4464 IP_PROTOCOL_ICMP, rx_fib_index0)))
4469 next0 = SNAT_IN2OUT_NEXT_DROP;
4470 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4474 snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
4476 key0.ext_host_addr = ip0->dst_address;
4477 key0.ext_host_port = 0;
4479 ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
4480 if (PREDICT_FALSE(!ses0))
4482 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4483 IP_PROTOCOL_ICMP, rx_fib_index0)))
4488 if (icmp0->type != ICMP4_echo_request)
4490 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4491 next0 = SNAT_IN2OUT_NEXT_DROP;
4494 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4496 key0.out_port = clib_host_to_net_u16 (lo_port0 +
4497 ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
4499 if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
4502 ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
4505 if (PREDICT_FALSE(!ses0))
4507 next0 = SNAT_IN2OUT_NEXT_DROP;
4508 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
4513 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
4514 !icmp_is_error_message (icmp0)))
4516 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4517 next0 = SNAT_IN2OUT_NEXT_DROP;
4521 u32 now = (u32) vlib_time_now (sm->vlib_main);
4523 ses0->state = SNAT_SESSION_ICMP_ACTIVE;
4524 ses0->expire = now + sm->icmp_timeout;
4527 *p_proto = protocol;
4530 p_value->addr = new_addr0;
4531 p_value->fib_index = sm->outside_fib_index;
4532 p_value->port = ses0->out.out_port;
4534 *p_dont_translate = dont_translate;
4536 *(snat_det_session_t**)d = ses0;
4538 *(snat_det_map_t**)e = dm0;
4542 /**********************/
4543 /*** worker handoff ***/
4544 /**********************/
4546 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
4547 vlib_node_runtime_t * node,
4548 vlib_frame_t * frame,
4551 snat_main_t *sm = &snat_main;
4552 vlib_thread_main_t *tm = vlib_get_thread_main ();
4553 u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
4554 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
4555 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
4557 vlib_frame_queue_elt_t *hf = 0;
4558 vlib_frame_queue_t *fq;
4559 vlib_frame_t *f = 0;
4561 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
4562 u32 next_worker_index = 0;
4563 u32 current_worker_index = ~0;
4564 u32 thread_index = vm->thread_index;
4567 vlib_frame_t *d = 0;
4569 ASSERT (vec_len (sm->workers));
4573 fq_index = sm->fq_in2out_output_index;
4574 to_node_index = sm->in2out_output_node_index;
4578 fq_index = sm->fq_in2out_index;
4579 to_node_index = sm->in2out_node_index;
4582 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
4584 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
4586 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
4587 tm->n_vlib_mains - 1,
4588 (vlib_frame_queue_t *) (~0));
4591 from = vlib_frame_vector_args (frame);
4592 n_left_from = frame->n_vectors;
4594 while (n_left_from > 0)
4607 b0 = vlib_get_buffer (vm, bi0);
4609 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
4610 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4612 ip0 = vlib_buffer_get_current (b0);
4614 next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
4616 if (PREDICT_FALSE (next_worker_index != thread_index))
4620 if (next_worker_index != current_worker_index)
4622 fq = is_vlib_frame_queue_congested (
4623 fq_index, next_worker_index, NAT_FQ_NELTS - 2,
4624 congested_handoff_queue_by_worker_index);
4628 /* if this is 1st frame */
4631 d = vlib_get_frame_to_node (vm, sm->error_node_index);
4632 to_next_drop = vlib_frame_vector_args (d);
4635 to_next_drop[0] = bi0;
4638 b0->error = node->errors[SNAT_IN2OUT_ERROR_FQ_CONGESTED];
4643 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4645 hf = vlib_get_worker_handoff_queue_elt (fq_index,
4647 handoff_queue_elt_by_worker_index);
4649 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
4650 to_next_worker = &hf->buffer_index[hf->n_vectors];
4651 current_worker_index = next_worker_index;
4654 /* enqueue to correct worker thread */
4655 to_next_worker[0] = bi0;
4657 n_left_to_next_worker--;
4659 if (n_left_to_next_worker == 0)
4661 hf->n_vectors = VLIB_FRAME_SIZE;
4662 vlib_put_frame_queue_elt (hf);
4663 current_worker_index = ~0;
4664 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
4671 /* if this is 1st frame */
4674 f = vlib_get_frame_to_node (vm, to_node_index);
4675 to_next = vlib_frame_vector_args (f);
4684 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
4685 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4687 snat_in2out_worker_handoff_trace_t *t =
4688 vlib_add_trace (vm, node, b0, sizeof (*t));
4689 t->next_worker_index = next_worker_index;
4690 t->do_handoff = do_handoff;
4695 vlib_put_frame_to_node (vm, to_node_index, f);
4698 vlib_put_frame_to_node (vm, sm->error_node_index, d);
4701 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4703 /* Ship frames to the worker nodes */
4704 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
4706 if (handoff_queue_elt_by_worker_index[i])
4708 hf = handoff_queue_elt_by_worker_index[i];
4710 * It works better to let the handoff node
4711 * rate-adapt, always ship the handoff queue element.
4713 if (1 || hf->n_vectors == hf->last_n_vectors)
4715 vlib_put_frame_queue_elt (hf);
4716 handoff_queue_elt_by_worker_index[i] = 0;
4719 hf->last_n_vectors = hf->n_vectors;
4721 congested_handoff_queue_by_worker_index[i] =
4722 (vlib_frame_queue_t *) (~0);
4725 current_worker_index = ~0;
4726 return frame->n_vectors;
4730 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
4731 vlib_node_runtime_t * node,
4732 vlib_frame_t * frame)
4734 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
4737 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
4738 .function = snat_in2out_worker_handoff_fn,
4739 .name = "nat44-in2out-worker-handoff",
4740 .vector_size = sizeof (u32),
4741 .format_trace = format_snat_in2out_worker_handoff_trace,
4742 .type = VLIB_NODE_TYPE_INTERNAL,
4744 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4745 .error_strings = snat_in2out_error_strings,
4754 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
4755 snat_in2out_worker_handoff_fn);
4758 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
4759 vlib_node_runtime_t * node,
4760 vlib_frame_t * frame)
4762 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
4765 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
4766 .function = snat_in2out_output_worker_handoff_fn,
4767 .name = "nat44-in2out-output-worker-handoff",
4768 .vector_size = sizeof (u32),
4769 .format_trace = format_snat_in2out_worker_handoff_trace,
4770 .type = VLIB_NODE_TYPE_INTERNAL,
4779 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
4780 snat_in2out_output_worker_handoff_fn);
4782 static_always_inline int
4783 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
4785 snat_address_t * ap;
4786 clib_bihash_kv_8_8_t kv, value;
4787 snat_session_key_t m_key;
4789 vec_foreach (ap, sm->addresses)
4791 if (ap->addr.as_u32 == dst_addr->as_u32)
4795 m_key.addr.as_u32 = dst_addr->as_u32;
4796 m_key.fib_index = 0;
4799 kv.key = m_key.as_u64;
4800 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4807 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
4808 vlib_node_runtime_t * node,
4809 vlib_frame_t * frame,
4812 u32 n_left_from, * from, * to_next, stats_node_index;
4813 snat_in2out_next_t next_index;
4814 u32 pkts_processed = 0;
4815 snat_main_t * sm = &snat_main;
4817 stats_node_index = is_ed ? nat44_ed_hairpin_dst_node.index :
4818 snat_hairpin_dst_node.index;
4820 from = vlib_frame_vector_args (frame);
4821 n_left_from = frame->n_vectors;
4822 next_index = node->cached_next_index;
4824 while (n_left_from > 0)
4828 vlib_get_next_frame (vm, node, next_index,
4829 to_next, n_left_to_next);
4831 while (n_left_from > 0 && n_left_to_next > 0)
4839 /* speculatively enqueue b0 to the current next frame */
4845 n_left_to_next -= 1;
4847 b0 = vlib_get_buffer (vm, bi0);
4848 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4849 ip0 = vlib_buffer_get_current (b0);
4851 proto0 = ip_proto_to_snat_proto (ip0->protocol);
4853 vnet_buffer (b0)->snat.flags = 0;
4854 if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
4856 if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
4858 udp_header_t * udp0 = ip4_next_header (ip0);
4859 tcp_header_t * tcp0 = (tcp_header_t *) udp0;
4861 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
4863 else if (proto0 == SNAT_PROTOCOL_ICMP)
4865 icmp46_header_t * icmp0 = ip4_next_header (ip0);
4867 snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
4872 nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
4874 nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
4877 vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
4880 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4882 /* verify speculative enqueue, maybe switch current next frame */
4883 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4884 to_next, n_left_to_next,
4888 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4891 vlib_node_increment_counter (vm, stats_node_index,
4892 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4894 return frame->n_vectors;
4898 snat_hairpin_dst_fn (vlib_main_t * vm,
4899 vlib_node_runtime_t * node,
4900 vlib_frame_t * frame)
4902 return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
4905 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
4906 .function = snat_hairpin_dst_fn,
4907 .name = "nat44-hairpin-dst",
4908 .vector_size = sizeof (u32),
4909 .type = VLIB_NODE_TYPE_INTERNAL,
4910 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4911 .error_strings = snat_in2out_error_strings,
4914 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4915 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4919 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
4920 snat_hairpin_dst_fn);
4923 nat44_ed_hairpin_dst_fn (vlib_main_t * vm,
4924 vlib_node_runtime_t * node,
4925 vlib_frame_t * frame)
4927 return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
4930 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
4931 .function = nat44_ed_hairpin_dst_fn,
4932 .name = "nat44-ed-hairpin-dst",
4933 .vector_size = sizeof (u32),
4934 .type = VLIB_NODE_TYPE_INTERNAL,
4935 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4936 .error_strings = snat_in2out_error_strings,
4939 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4940 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4944 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_dst_node,
4945 nat44_ed_hairpin_dst_fn);
4948 snat_hairpin_src_fn_inline (vlib_main_t * vm,
4949 vlib_node_runtime_t * node,
4950 vlib_frame_t * frame,
4953 u32 n_left_from, * from, * to_next, stats_node_index;
4954 snat_in2out_next_t next_index;
4955 u32 pkts_processed = 0;
4956 snat_main_t *sm = &snat_main;
4958 stats_node_index = is_ed ? nat44_ed_hairpin_src_node.index :
4959 snat_hairpin_src_node.index;
4961 from = vlib_frame_vector_args (frame);
4962 n_left_from = frame->n_vectors;
4963 next_index = node->cached_next_index;
4965 while (n_left_from > 0)
4969 vlib_get_next_frame (vm, node, next_index,
4970 to_next, n_left_to_next);
4972 while (n_left_from > 0 && n_left_to_next > 0)
4977 snat_interface_t *i;
4980 /* speculatively enqueue b0 to the current next frame */
4986 n_left_to_next -= 1;
4988 b0 = vlib_get_buffer (vm, bi0);
4989 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4990 next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
4992 pool_foreach (i, sm->output_feature_interfaces,
4994 /* Only packets from NAT inside interface */
4995 if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
4997 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
4998 SNAT_FLAG_HAIRPINNING))
5000 if (PREDICT_TRUE (sm->num_workers > 1))
5001 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
5003 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
5009 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5011 /* verify speculative enqueue, maybe switch current next frame */
5012 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5013 to_next, n_left_to_next,
5017 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5020 vlib_node_increment_counter (vm, stats_node_index,
5021 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5023 return frame->n_vectors;
5027 snat_hairpin_src_fn (vlib_main_t * vm,
5028 vlib_node_runtime_t * node,
5029 vlib_frame_t * frame)
5031 return snat_hairpin_src_fn_inline (vm, node, frame, 0);
5034 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
5035 .function = snat_hairpin_src_fn,
5036 .name = "nat44-hairpin-src",
5037 .vector_size = sizeof (u32),
5038 .type = VLIB_NODE_TYPE_INTERNAL,
5039 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5040 .error_strings = snat_in2out_error_strings,
5041 .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5043 [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5044 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
5045 [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5046 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5050 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
5051 snat_hairpin_src_fn);
5054 nat44_ed_hairpin_src_fn (vlib_main_t * vm,
5055 vlib_node_runtime_t * node,
5056 vlib_frame_t * frame)
5058 return snat_hairpin_src_fn_inline (vm, node, frame, 1);
5061 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
5062 .function = nat44_ed_hairpin_src_fn,
5063 .name = "nat44-ed-hairpin-src",
5064 .vector_size = sizeof (u32),
5065 .type = VLIB_NODE_TYPE_INTERNAL,
5066 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5067 .error_strings = snat_in2out_error_strings,
5068 .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5070 [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5071 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
5072 [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5073 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5077 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_src_node,
5078 nat44_ed_hairpin_src_fn);
5081 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
5082 vlib_node_runtime_t * node,
5083 vlib_frame_t * frame)
5085 u32 n_left_from, * from, * to_next;
5086 snat_in2out_next_t next_index;
5087 u32 pkts_processed = 0;
5088 snat_main_t * sm = &snat_main;
5089 u32 stats_node_index;
5091 stats_node_index = snat_in2out_fast_node.index;
5093 from = vlib_frame_vector_args (frame);
5094 n_left_from = frame->n_vectors;
5095 next_index = node->cached_next_index;
5097 while (n_left_from > 0)
5101 vlib_get_next_frame (vm, node, next_index,
5102 to_next, n_left_to_next);
5104 while (n_left_from > 0 && n_left_to_next > 0)
5112 u32 new_addr0, old_addr0;
5113 u16 old_port0, new_port0;
5114 udp_header_t * udp0;
5115 tcp_header_t * tcp0;
5116 icmp46_header_t * icmp0;
5117 snat_session_key_t key0, sm0;
5121 /* speculatively enqueue b0 to the current next frame */
5127 n_left_to_next -= 1;
5129 b0 = vlib_get_buffer (vm, bi0);
5130 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5132 ip0 = vlib_buffer_get_current (b0);
5133 udp0 = ip4_next_header (ip0);
5134 tcp0 = (tcp_header_t *) udp0;
5135 icmp0 = (icmp46_header_t *) udp0;
5137 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5138 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
5140 if (PREDICT_FALSE(ip0->ttl == 1))
5142 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
5143 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
5144 ICMP4_time_exceeded_ttl_exceeded_in_transit,
5146 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
5150 proto0 = ip_proto_to_snat_proto (ip0->protocol);
5152 if (PREDICT_FALSE (proto0 == ~0))
5155 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
5157 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
5158 rx_fib_index0, node, next0, ~0, 0, 0);
5162 key0.addr = ip0->src_address;
5163 key0.protocol = proto0;
5164 key0.port = udp0->src_port;
5165 key0.fib_index = rx_fib_index0;
5167 if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
5169 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
5170 next0= SNAT_IN2OUT_NEXT_DROP;
5174 new_addr0 = sm0.addr.as_u32;
5175 new_port0 = sm0.port;
5176 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
5177 old_addr0 = ip0->src_address.as_u32;
5178 ip0->src_address.as_u32 = new_addr0;
5180 sum0 = ip0->checksum;
5181 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5183 src_address /* changed member */);
5184 ip0->checksum = ip_csum_fold (sum0);
5186 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
5188 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5190 old_port0 = tcp0->src_port;
5191 tcp0->src_port = new_port0;
5193 sum0 = tcp0->checksum;
5194 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5196 dst_address /* changed member */);
5197 sum0 = ip_csum_update (sum0, old_port0, new_port0,
5198 ip4_header_t /* cheat */,
5199 length /* changed member */);
5200 tcp0->checksum = ip_csum_fold(sum0);
5204 old_port0 = udp0->src_port;
5205 udp0->src_port = new_port0;
5211 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5213 sum0 = tcp0->checksum;
5214 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5216 dst_address /* changed member */);
5217 tcp0->checksum = ip_csum_fold(sum0);
5222 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
5225 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
5226 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
5228 snat_in2out_trace_t *t =
5229 vlib_add_trace (vm, node, b0, sizeof (*t));
5230 t->sw_if_index = sw_if_index0;
5231 t->next_index = next0;
5234 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5236 /* verify speculative enqueue, maybe switch current next frame */
5237 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5238 to_next, n_left_to_next,
5242 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5245 vlib_node_increment_counter (vm, stats_node_index,
5246 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5248 return frame->n_vectors;
5252 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
5253 .function = snat_in2out_fast_static_map_fn,
5254 .name = "nat44-in2out-fast",
5255 .vector_size = sizeof (u32),
5256 .format_trace = format_snat_in2out_fast_trace,
5257 .type = VLIB_NODE_TYPE_INTERNAL,
5259 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5260 .error_strings = snat_in2out_error_strings,
5262 .runtime_data_bytes = sizeof (snat_runtime_t),
5264 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
5266 /* edit / add dispositions here */
5268 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5269 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5270 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
5271 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
5272 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
5276 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);