2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/fib/ip4_fib.h>
25 #include <nat/nat_ipfix_logging.h>
26 #include <nat/nat_det.h>
27 #include <nat/nat_reass.h>
28 #include <nat/nat_inlines.h>
30 #include <vppinfra/hash.h>
31 #include <vppinfra/error.h>
32 #include <vppinfra/elog.h>
39 } snat_in2out_trace_t;
42 u32 next_worker_index;
44 } snat_in2out_worker_handoff_trace_t;
46 /* packet trace format function */
47 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
49 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
50 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
51 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
54 tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
56 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
57 t->sw_if_index, t->next_index, t->session_index);
62 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
64 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
65 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
66 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
68 s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
69 t->sw_if_index, t->next_index);
74 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
76 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
77 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
78 snat_in2out_worker_handoff_trace_t * t =
79 va_arg (*args, snat_in2out_worker_handoff_trace_t *);
82 m = t->do_handoff ? "next worker" : "same worker";
83 s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
92 } nat44_in2out_reass_trace_t;
94 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
96 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
97 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
98 nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
100 s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
101 t->sw_if_index, t->next_index,
102 t->cached ? "cached" : "translated");
107 vlib_node_registration_t snat_in2out_node;
108 vlib_node_registration_t snat_in2out_slowpath_node;
109 vlib_node_registration_t snat_in2out_fast_node;
110 vlib_node_registration_t snat_in2out_worker_handoff_node;
111 vlib_node_registration_t snat_det_in2out_node;
112 vlib_node_registration_t snat_in2out_output_node;
113 vlib_node_registration_t snat_in2out_output_slowpath_node;
114 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
115 vlib_node_registration_t snat_hairpin_dst_node;
116 vlib_node_registration_t snat_hairpin_src_node;
117 vlib_node_registration_t nat44_hairpinning_node;
118 vlib_node_registration_t nat44_in2out_reass_node;
119 vlib_node_registration_t nat44_ed_in2out_node;
120 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
121 vlib_node_registration_t nat44_ed_in2out_output_node;
122 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
123 vlib_node_registration_t nat44_ed_hairpin_dst_node;
124 vlib_node_registration_t nat44_ed_hairpin_src_node;
125 vlib_node_registration_t nat44_ed_hairpinning_node;
127 #define foreach_snat_in2out_error \
128 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
129 _(IN2OUT_PACKETS, "Good in2out packets processed") \
130 _(OUT_OF_PORTS, "Out of ports") \
131 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \
132 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
133 _(NO_TRANSLATION, "No translation") \
134 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
135 _(DROP_FRAGMENT, "Drop fragment") \
136 _(MAX_REASS, "Maximum reassemblies exceeded") \
137 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
138 _(FQ_CONGESTED, "Handoff frame queue congested")
141 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
142 foreach_snat_in2out_error
145 } snat_in2out_error_t;
147 static char * snat_in2out_error_strings[] = {
148 #define _(sym,string) string,
149 foreach_snat_in2out_error
154 SNAT_IN2OUT_NEXT_LOOKUP,
155 SNAT_IN2OUT_NEXT_DROP,
156 SNAT_IN2OUT_NEXT_ICMP_ERROR,
157 SNAT_IN2OUT_NEXT_SLOW_PATH,
158 SNAT_IN2OUT_NEXT_REASS,
160 } snat_in2out_next_t;
163 SNAT_HAIRPIN_SRC_NEXT_DROP,
164 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
165 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
166 SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
167 SNAT_HAIRPIN_SRC_N_NEXT,
168 } snat_hairpin_next_t;
171 * @brief Check if packet should be translated
173 * Packets aimed at outside interface and external address with active session
174 * should be translated.
177 * @param rt NAT runtime data
178 * @param sw_if_index0 index of the inside interface
179 * @param ip0 IPv4 header
180 * @param proto0 NAT protocol
181 * @param rx_fib_index0 RX FIB index
183 * @returns 0 if packet should be translated otherwise 1
186 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
187 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
193 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
194 nat_outside_fib_t *outside_fib;
196 .fp_proto = FIB_PROTOCOL_IP4,
199 .ip4.as_u32 = ip0->dst_address.as_u32,
203 /* Don't NAT packet aimed at the intfc address */
204 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
205 ip0->dst_address.as_u32)))
208 fei = fib_table_lookup (rx_fib_index0, &pfx);
209 if (FIB_NODE_INDEX_INVALID != fei)
211 u32 sw_if_index = fib_entry_get_resolving_interface (fei);
212 if (sw_if_index == ~0)
214 vec_foreach (outside_fib, sm->outside_fibs)
216 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
217 if (FIB_NODE_INDEX_INVALID != fei)
219 sw_if_index = fib_entry_get_resolving_interface (fei);
220 if (sw_if_index != ~0)
225 if (sw_if_index == ~0)
229 pool_foreach (i, sm->interfaces,
231 /* NAT packet aimed at outside interface */
232 if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
241 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
242 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
243 u32 rx_fib_index0, u32 thread_index)
245 udp_header_t * udp0 = ip4_next_header (ip0);
246 snat_session_key_t key0, sm0;
247 clib_bihash_kv_8_8_t kv0, value0;
249 key0.addr = ip0->dst_address;
250 key0.port = udp0->dst_port;
251 key0.protocol = proto0;
252 key0.fib_index = sm->outside_fib_index;
253 kv0.key = key0.as_u64;
255 /* NAT packet aimed at external address if */
256 /* has active sessions */
257 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
260 /* or is static mappings */
261 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
267 if (sm->forwarding_enabled)
270 return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
275 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
276 u32 proto0, u16 src_port, u16 dst_port,
277 u32 thread_index, u32 sw_if_index)
279 snat_session_key_t key0;
280 clib_bihash_kv_8_8_t kv0, value0;
284 key0.addr = ip0->src_address;
285 key0.port = src_port;
286 key0.protocol = proto0;
287 key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
288 kv0.key = key0.as_u64;
290 if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
295 key0.addr = ip0->dst_address;
296 key0.port = dst_port;
297 key0.protocol = proto0;
298 kv0.key = key0.as_u64;
299 if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
303 pool_foreach (i, sm->output_feature_interfaces,
305 if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
314 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
317 snat_session_key_t * key0,
318 snat_session_t ** sessionp,
319 vlib_node_runtime_t * node,
325 clib_bihash_kv_8_8_t kv0;
326 snat_session_key_t key1;
327 u32 address_index = ~0;
328 udp_header_t * udp0 = ip4_next_header (ip0);
330 nat_outside_fib_t *outside_fib;
331 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
333 .fp_proto = FIB_PROTOCOL_IP4,
336 .ip4.as_u32 = ip0->dst_address.as_u32,
340 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
342 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
343 nat_ipfix_logging_max_sessions(sm->max_translations);
344 nat_log_notice ("maximum sessions exceeded");
345 return SNAT_IN2OUT_NEXT_DROP;
348 key1.protocol = key0->protocol;
350 u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
354 nat_log_warn ("create NAT user failed");
355 return SNAT_IN2OUT_NEXT_DROP;
358 /* First try to match static mapping by local address and port */
359 if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
361 /* Try to create dynamic translation */
362 if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
366 sm->per_thread_data[thread_index].snat_thread_index))
368 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
369 return SNAT_IN2OUT_NEXT_DROP;
375 s = nat_session_alloc_or_recycle (sm, u, thread_index);
378 nat_log_warn ("create NAT session failed");
379 return SNAT_IN2OUT_NEXT_DROP;
383 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
384 user_session_increment (sm, u, is_sm);
385 s->outside_address_index = address_index;
388 s->out2in.protocol = key0->protocol;
389 s->out2in.fib_index = sm->outside_fib_index;
390 switch (vec_len (sm->outside_fibs))
393 s->out2in.fib_index = sm->outside_fib_index;
396 s->out2in.fib_index = sm->outside_fibs[0].fib_index;
399 vec_foreach (outside_fib, sm->outside_fibs)
401 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
402 if (FIB_NODE_INDEX_INVALID != fei)
404 if (fib_entry_get_resolving_interface (fei) != ~0)
406 s->out2in.fib_index = outside_fib->fib_index;
413 s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
414 s->ext_host_port = udp0->dst_port;
417 /* Add to translation hashes */
418 kv0.key = s->in2out.as_u64;
419 kv0.value = s - sm->per_thread_data[thread_index].sessions;
420 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
422 nat_log_notice ("in2out key add failed");
424 kv0.key = s->out2in.as_u64;
425 kv0.value = s - sm->per_thread_data[thread_index].sessions;
427 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
429 nat_log_notice ("out2in key add failed");
432 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
433 s->out2in.addr.as_u32,
437 s->in2out.fib_index);
442 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
443 snat_session_key_t *p_key0)
445 icmp46_header_t *icmp0;
446 snat_session_key_t key0;
447 icmp_echo_header_t *echo0, *inner_echo0 = 0;
448 ip4_header_t *inner_ip0 = 0;
450 icmp46_header_t *inner_icmp0;
452 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
453 echo0 = (icmp_echo_header_t *)(icmp0+1);
455 if (!icmp_is_error_message (icmp0))
457 key0.protocol = SNAT_PROTOCOL_ICMP;
458 key0.addr = ip0->src_address;
459 key0.port = echo0->identifier;
463 inner_ip0 = (ip4_header_t *)(echo0+1);
464 l4_header = ip4_next_header (inner_ip0);
465 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
466 key0.addr = inner_ip0->dst_address;
467 switch (key0.protocol)
469 case SNAT_PROTOCOL_ICMP:
470 inner_icmp0 = (icmp46_header_t*)l4_header;
471 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
472 key0.port = inner_echo0->identifier;
474 case SNAT_PROTOCOL_UDP:
475 case SNAT_PROTOCOL_TCP:
476 key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
479 return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
483 return -1; /* success */
487 * Get address and port values to be used for ICMP packet translation
488 * and create session if needed
490 * @param[in,out] sm NAT main
491 * @param[in,out] node NAT node runtime
492 * @param[in] thread_index thread index
493 * @param[in,out] b0 buffer containing packet to be translated
494 * @param[out] p_proto protocol used for matching
495 * @param[out] p_value address and port after NAT translation
496 * @param[out] p_dont_translate if packet should not be translated
497 * @param d optional parameter
498 * @param e optional parameter
500 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
501 u32 thread_index, vlib_buffer_t *b0,
502 ip4_header_t *ip0, u8 *p_proto,
503 snat_session_key_t *p_value,
504 u8 *p_dont_translate, void *d, void *e)
506 icmp46_header_t *icmp0;
509 snat_session_key_t key0;
510 snat_session_t *s0 = 0;
511 u8 dont_translate = 0;
512 clib_bihash_kv_8_8_t kv0, value0;
516 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
517 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
518 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
520 err = icmp_get_key (ip0, &key0);
523 b0->error = node->errors[err];
524 next0 = SNAT_IN2OUT_NEXT_DROP;
527 key0.fib_index = rx_fib_index0;
529 kv0.key = key0.as_u64;
531 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
534 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
536 if (PREDICT_FALSE(nat_not_translate_output_feature(sm, ip0,
537 key0.protocol, key0.port, key0.port, thread_index, sw_if_index0)))
545 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
546 ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
553 if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
555 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
556 next0 = SNAT_IN2OUT_NEXT_DROP;
560 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
561 &s0, node, next0, thread_index);
563 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
568 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
569 icmp0->type != ICMP4_echo_reply &&
570 !icmp_is_error_message (icmp0)))
572 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
573 next0 = SNAT_IN2OUT_NEXT_DROP;
577 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
582 *p_proto = key0.protocol;
584 *p_value = s0->out2in;
585 *p_dont_translate = dont_translate;
587 *(snat_session_t**)d = s0;
592 * Get address and port values to be used for ICMP packet translation
594 * @param[in] sm NAT main
595 * @param[in,out] node NAT node runtime
596 * @param[in] thread_index thread index
597 * @param[in,out] b0 buffer containing packet to be translated
598 * @param[out] p_proto protocol used for matching
599 * @param[out] p_value address and port after NAT translation
600 * @param[out] p_dont_translate if packet should not be translated
601 * @param d optional parameter
602 * @param e optional parameter
604 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
605 u32 thread_index, vlib_buffer_t *b0,
606 ip4_header_t *ip0, u8 *p_proto,
607 snat_session_key_t *p_value,
608 u8 *p_dont_translate, void *d, void *e)
610 icmp46_header_t *icmp0;
613 snat_session_key_t key0;
614 snat_session_key_t sm0;
615 u8 dont_translate = 0;
620 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
621 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
622 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
624 err = icmp_get_key (ip0, &key0);
627 b0->error = node->errors[err];
628 next0 = SNAT_IN2OUT_NEXT_DROP;
631 key0.fib_index = rx_fib_index0;
633 if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
635 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
636 IP_PROTOCOL_ICMP, rx_fib_index0)))
642 if (icmp_is_error_message (icmp0))
644 next0 = SNAT_IN2OUT_NEXT_DROP;
648 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
649 next0 = SNAT_IN2OUT_NEXT_DROP;
653 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
654 (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
655 !icmp_is_error_message (icmp0)))
657 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
658 next0 = SNAT_IN2OUT_NEXT_DROP;
665 *p_proto = key0.protocol;
666 *p_dont_translate = dont_translate;
670 static inline u32 icmp_in2out (snat_main_t *sm,
673 icmp46_header_t * icmp0,
676 vlib_node_runtime_t * node,
682 snat_session_key_t sm0;
684 icmp_echo_header_t *echo0, *inner_echo0 = 0;
685 ip4_header_t *inner_ip0;
687 icmp46_header_t *inner_icmp0;
689 u32 new_addr0, old_addr0;
690 u16 old_id0, new_id0;
695 echo0 = (icmp_echo_header_t *)(icmp0+1);
697 next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
698 &protocol, &sm0, &dont_translate, d, e);
701 if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
704 sum0 = ip_incremental_checksum (0, icmp0,
705 ntohs(ip0->length) - ip4_header_bytes (ip0));
706 checksum0 = ~ip_csum_fold (sum0);
707 if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
709 next0 = SNAT_IN2OUT_NEXT_DROP;
713 old_addr0 = ip0->src_address.as_u32;
714 new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
715 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
716 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
718 sum0 = ip0->checksum;
719 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
720 src_address /* changed member */);
721 ip0->checksum = ip_csum_fold (sum0);
723 if (icmp0->checksum == 0)
724 icmp0->checksum = 0xffff;
726 if (!icmp_is_error_message (icmp0))
729 if (PREDICT_FALSE(new_id0 != echo0->identifier))
731 old_id0 = echo0->identifier;
733 echo0->identifier = new_id0;
735 sum0 = icmp0->checksum;
736 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
738 icmp0->checksum = ip_csum_fold (sum0);
743 inner_ip0 = (ip4_header_t *)(echo0+1);
744 l4_header = ip4_next_header (inner_ip0);
746 if (!ip4_header_checksum_is_valid (inner_ip0))
748 next0 = SNAT_IN2OUT_NEXT_DROP;
752 old_addr0 = inner_ip0->dst_address.as_u32;
753 inner_ip0->dst_address = sm0.addr;
754 new_addr0 = inner_ip0->dst_address.as_u32;
756 sum0 = icmp0->checksum;
757 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
758 dst_address /* changed member */);
759 icmp0->checksum = ip_csum_fold (sum0);
763 case SNAT_PROTOCOL_ICMP:
764 inner_icmp0 = (icmp46_header_t*)l4_header;
765 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
767 old_id0 = inner_echo0->identifier;
769 inner_echo0->identifier = new_id0;
771 sum0 = icmp0->checksum;
772 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
774 icmp0->checksum = ip_csum_fold (sum0);
776 case SNAT_PROTOCOL_UDP:
777 case SNAT_PROTOCOL_TCP:
778 old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
780 ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
782 sum0 = icmp0->checksum;
783 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
785 icmp0->checksum = ip_csum_fold (sum0);
799 * Hairpinning allows two endpoints on the internal side of the NAT to
800 * communicate even if they only use each other's external IP addresses
803 * @param sm NAT main.
804 * @param b0 Vlib buffer.
805 * @param ip0 IP header.
806 * @param udp0 UDP header.
807 * @param tcp0 TCP header.
808 * @param proto0 NAT protocol.
811 snat_hairpinning (snat_main_t *sm,
819 snat_session_key_t key0, sm0;
821 clib_bihash_kv_8_8_t kv0, value0;
823 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
824 u16 new_dst_port0, old_dst_port0;
827 key0.addr = ip0->dst_address;
828 key0.port = udp0->dst_port;
829 key0.protocol = proto0;
830 key0.fib_index = sm->outside_fib_index;
831 kv0.key = key0.as_u64;
833 /* Check if destination is static mappings */
834 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
836 new_dst_addr0 = sm0.addr.as_u32;
837 new_dst_port0 = sm0.port;
838 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
840 /* or active session */
843 if (sm->num_workers > 1)
844 ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
846 ti = sm->num_workers;
850 clib_bihash_kv_16_8_t ed_kv, ed_value;
851 make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
852 ip0->protocol, sm->outside_fib_index, udp0->dst_port,
854 rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
860 rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
867 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
868 new_dst_addr0 = s0->in2out.addr.as_u32;
869 new_dst_port0 = s0->in2out.port;
870 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
873 /* Destination is behind the same NAT, use internal address and port */
876 old_dst_addr0 = ip0->dst_address.as_u32;
877 ip0->dst_address.as_u32 = new_dst_addr0;
878 sum0 = ip0->checksum;
879 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
880 ip4_header_t, dst_address);
881 ip0->checksum = ip_csum_fold (sum0);
883 old_dst_port0 = tcp0->dst;
884 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
886 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
888 tcp0->dst = new_dst_port0;
889 sum0 = tcp0->checksum;
890 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
891 ip4_header_t, dst_address);
892 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
893 ip4_header_t /* cheat */, length);
894 tcp0->checksum = ip_csum_fold(sum0);
898 udp0->dst_port = new_dst_port0;
904 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
906 sum0 = tcp0->checksum;
907 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
908 ip4_header_t, dst_address);
909 tcp0->checksum = ip_csum_fold(sum0);
918 snat_icmp_hairpinning (snat_main_t *sm,
921 icmp46_header_t * icmp0,
924 snat_session_key_t key0, sm0;
925 clib_bihash_kv_8_8_t kv0, value0;
926 u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
931 if (!icmp_is_error_message (icmp0))
933 icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
934 u16 icmp_id0 = echo0->identifier;
935 key0.addr = ip0->dst_address;
936 key0.port = icmp_id0;
937 key0.protocol = SNAT_PROTOCOL_ICMP;
938 key0.fib_index = sm->outside_fib_index;
939 kv0.key = key0.as_u64;
941 if (sm->num_workers > 1)
942 ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
944 ti = sm->num_workers;
946 /* Check if destination is in active sessions */
949 clib_bihash_kv_16_8_t ed_kv, ed_value;
950 make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
951 IP_PROTOCOL_ICMP, sm->outside_fib_index, icmp_id0, 0);
952 rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
958 rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
964 /* or static mappings */
965 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
967 new_dst_addr0 = sm0.addr.as_u32;
968 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
973 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
974 new_dst_addr0 = s0->in2out.addr.as_u32;
975 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
976 echo0->identifier = s0->in2out.port;
977 sum0 = icmp0->checksum;
978 sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
979 icmp_echo_header_t, identifier);
980 icmp0->checksum = ip_csum_fold (sum0);
983 /* Destination is behind the same NAT, use internal address and port */
986 old_dst_addr0 = ip0->dst_address.as_u32;
987 ip0->dst_address.as_u32 = new_dst_addr0;
988 sum0 = ip0->checksum;
989 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
990 ip4_header_t, dst_address);
991 ip0->checksum = ip_csum_fold (sum0);
997 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1000 icmp46_header_t * icmp0,
1003 vlib_node_runtime_t * node,
1007 snat_session_t ** p_s0)
1009 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1010 next0, thread_index, p_s0, 0);
1011 snat_session_t * s0 = *p_s0;
1012 if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1015 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1016 snat_icmp_hairpinning(sm, b0, ip0, icmp0, sm->endpoint_dependent);
1018 nat44_session_update_counters (s0, now,
1019 vlib_buffer_length_in_chain (sm->vlib_main, b0));
1020 /* Per-user LRU list maintenance */
1021 nat44_session_update_lru (sm, s0, thread_index);
1027 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
1031 clib_bihash_kv_8_8_t kv, value;
1032 snat_static_mapping_t *m;
1033 u32 old_addr, new_addr;
1036 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1037 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1040 m = pool_elt_at_index (sm->static_mappings, value.value);
1042 old_addr = ip->dst_address.as_u32;
1043 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1045 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1046 ip->checksum = ip_csum_fold (sum);
1048 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1049 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1053 nat_in2out_sm_unknown_proto (snat_main_t *sm,
1058 clib_bihash_kv_8_8_t kv, value;
1059 snat_static_mapping_t *m;
1060 snat_session_key_t m_key;
1061 u32 old_addr, new_addr;
1064 m_key.addr = ip->src_address;
1067 m_key.fib_index = rx_fib_index;
1068 kv.key = m_key.as_u64;
1069 if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1072 m = pool_elt_at_index (sm->static_mappings, value.value);
1074 old_addr = ip->src_address.as_u32;
1075 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1077 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1078 ip->checksum = ip_csum_fold (sum);
1082 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1084 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1085 nat_hairpinning_sm_unknown_proto (sm, b, ip);
1092 snat_in2out_node_fn_inline (vlib_main_t * vm,
1093 vlib_node_runtime_t * node,
1094 vlib_frame_t * frame, int is_slow_path,
1095 int is_output_feature)
1097 u32 n_left_from, * from, * to_next;
1098 snat_in2out_next_t next_index;
1099 u32 pkts_processed = 0;
1100 snat_main_t * sm = &snat_main;
1101 f64 now = vlib_time_now (vm);
1102 u32 stats_node_index;
1103 u32 thread_index = vm->thread_index;
1105 stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1106 snat_in2out_node.index;
1108 from = vlib_frame_vector_args (frame);
1109 n_left_from = frame->n_vectors;
1110 next_index = node->cached_next_index;
1112 while (n_left_from > 0)
1116 vlib_get_next_frame (vm, node, next_index,
1117 to_next, n_left_to_next);
1119 while (n_left_from >= 4 && n_left_to_next >= 2)
1122 vlib_buffer_t * b0, * b1;
1124 u32 sw_if_index0, sw_if_index1;
1125 ip4_header_t * ip0, * ip1;
1126 ip_csum_t sum0, sum1;
1127 u32 new_addr0, old_addr0, new_addr1, old_addr1;
1128 u16 old_port0, new_port0, old_port1, new_port1;
1129 udp_header_t * udp0, * udp1;
1130 tcp_header_t * tcp0, * tcp1;
1131 icmp46_header_t * icmp0, * icmp1;
1132 snat_session_key_t key0, key1;
1133 u32 rx_fib_index0, rx_fib_index1;
1135 snat_session_t * s0 = 0, * s1 = 0;
1136 clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1137 u32 iph_offset0 = 0, iph_offset1 = 0;
1139 /* Prefetch next iteration. */
1141 vlib_buffer_t * p2, * p3;
1143 p2 = vlib_get_buffer (vm, from[2]);
1144 p3 = vlib_get_buffer (vm, from[3]);
1146 vlib_prefetch_buffer_header (p2, LOAD);
1147 vlib_prefetch_buffer_header (p3, LOAD);
1149 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1150 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1153 /* speculatively enqueue b0 and b1 to the current next frame */
1154 to_next[0] = bi0 = from[0];
1155 to_next[1] = bi1 = from[1];
1159 n_left_to_next -= 2;
1161 b0 = vlib_get_buffer (vm, bi0);
1162 b1 = vlib_get_buffer (vm, bi1);
1164 if (is_output_feature)
1165 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1167 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1170 udp0 = ip4_next_header (ip0);
1171 tcp0 = (tcp_header_t *) udp0;
1172 icmp0 = (icmp46_header_t *) udp0;
1174 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1175 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1178 next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1180 if (PREDICT_FALSE(ip0->ttl == 1))
1182 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1183 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1184 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1186 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1190 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1192 /* Next configured feature, probably ip4-lookup */
1195 if (PREDICT_FALSE (proto0 == ~0))
1197 if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1199 next0 = SNAT_IN2OUT_NEXT_DROP;
1200 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1205 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1207 next0 = icmp_in2out_slow_path
1208 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1209 node, next0, now, thread_index, &s0);
1215 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1217 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1221 if (ip4_is_fragment (ip0))
1223 next0 = SNAT_IN2OUT_NEXT_REASS;
1228 key0.addr = ip0->src_address;
1229 key0.port = udp0->src_port;
1230 key0.protocol = proto0;
1231 key0.fib_index = rx_fib_index0;
1233 kv0.key = key0.as_u64;
1235 if (PREDICT_FALSE (clib_bihash_search_8_8 (
1236 &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1240 if (is_output_feature)
1242 if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1243 ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1248 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1249 ip0, proto0, rx_fib_index0, thread_index)))
1253 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1254 &s0, node, next0, thread_index);
1255 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1260 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1265 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1268 b0->flags |= VNET_BUFFER_F_IS_NATED;
1270 old_addr0 = ip0->src_address.as_u32;
1271 ip0->src_address = s0->out2in.addr;
1272 new_addr0 = ip0->src_address.as_u32;
1273 if (!is_output_feature)
1274 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1276 sum0 = ip0->checksum;
1277 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1279 src_address /* changed member */);
1280 ip0->checksum = ip_csum_fold (sum0);
1282 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1284 old_port0 = tcp0->src_port;
1285 tcp0->src_port = s0->out2in.port;
1286 new_port0 = tcp0->src_port;
1288 sum0 = tcp0->checksum;
1289 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1291 dst_address /* changed member */);
1292 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1293 ip4_header_t /* cheat */,
1294 length /* changed member */);
1295 tcp0->checksum = ip_csum_fold(sum0);
1299 old_port0 = udp0->src_port;
1300 udp0->src_port = s0->out2in.port;
1305 nat44_session_update_counters (s0, now,
1306 vlib_buffer_length_in_chain (vm, b0));
1307 /* Per-user LRU list maintenance */
1308 nat44_session_update_lru (sm, s0, thread_index);
1311 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1312 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1314 snat_in2out_trace_t *t =
1315 vlib_add_trace (vm, node, b0, sizeof (*t));
1316 t->is_slow_path = is_slow_path;
1317 t->sw_if_index = sw_if_index0;
1318 t->next_index = next0;
1319 t->session_index = ~0;
1321 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1324 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1326 if (is_output_feature)
1327 iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1329 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1332 udp1 = ip4_next_header (ip1);
1333 tcp1 = (tcp_header_t *) udp1;
1334 icmp1 = (icmp46_header_t *) udp1;
1336 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1337 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1340 if (PREDICT_FALSE(ip1->ttl == 1))
1342 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1343 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1344 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1346 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1350 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1352 /* Next configured feature, probably ip4-lookup */
1355 if (PREDICT_FALSE (proto1 == ~0))
1357 if (nat_in2out_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1359 next1 = SNAT_IN2OUT_NEXT_DROP;
1360 b1->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1365 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1367 next1 = icmp_in2out_slow_path
1368 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1369 next1, now, thread_index, &s1);
1375 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1377 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1381 if (ip4_is_fragment (ip1))
1383 next1 = SNAT_IN2OUT_NEXT_REASS;
1388 key1.addr = ip1->src_address;
1389 key1.port = udp1->src_port;
1390 key1.protocol = proto1;
1391 key1.fib_index = rx_fib_index1;
1393 kv1.key = key1.as_u64;
1395 if (PREDICT_FALSE(clib_bihash_search_8_8 (
1396 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1400 if (is_output_feature)
1402 if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1403 ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1408 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1409 ip1, proto1, rx_fib_index1, thread_index)))
1413 next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1414 &s1, node, next1, thread_index);
1415 if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1420 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1425 s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1428 b1->flags |= VNET_BUFFER_F_IS_NATED;
1430 old_addr1 = ip1->src_address.as_u32;
1431 ip1->src_address = s1->out2in.addr;
1432 new_addr1 = ip1->src_address.as_u32;
1433 if (!is_output_feature)
1434 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1436 sum1 = ip1->checksum;
1437 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1439 src_address /* changed member */);
1440 ip1->checksum = ip_csum_fold (sum1);
1442 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1444 old_port1 = tcp1->src_port;
1445 tcp1->src_port = s1->out2in.port;
1446 new_port1 = tcp1->src_port;
1448 sum1 = tcp1->checksum;
1449 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1451 dst_address /* changed member */);
1452 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1453 ip4_header_t /* cheat */,
1454 length /* changed member */);
1455 tcp1->checksum = ip_csum_fold(sum1);
1459 old_port1 = udp1->src_port;
1460 udp1->src_port = s1->out2in.port;
1465 nat44_session_update_counters (s1, now,
1466 vlib_buffer_length_in_chain (vm, b1));
1467 /* Per-user LRU list maintenance */
1468 nat44_session_update_lru (sm, s1, thread_index);
1471 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1472 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1474 snat_in2out_trace_t *t =
1475 vlib_add_trace (vm, node, b1, sizeof (*t));
1476 t->sw_if_index = sw_if_index1;
1477 t->next_index = next1;
1478 t->session_index = ~0;
1480 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1483 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1485 /* verify speculative enqueues, maybe switch current next frame */
1486 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1487 to_next, n_left_to_next,
1488 bi0, bi1, next0, next1);
1491 while (n_left_from > 0 && n_left_to_next > 0)
1499 u32 new_addr0, old_addr0;
1500 u16 old_port0, new_port0;
1501 udp_header_t * udp0;
1502 tcp_header_t * tcp0;
1503 icmp46_header_t * icmp0;
1504 snat_session_key_t key0;
1507 snat_session_t * s0 = 0;
1508 clib_bihash_kv_8_8_t kv0, value0;
1509 u32 iph_offset0 = 0;
1511 /* speculatively enqueue b0 to the current next frame */
1517 n_left_to_next -= 1;
1519 b0 = vlib_get_buffer (vm, bi0);
1520 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1522 if (is_output_feature)
1523 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1525 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1528 udp0 = ip4_next_header (ip0);
1529 tcp0 = (tcp_header_t *) udp0;
1530 icmp0 = (icmp46_header_t *) udp0;
1532 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1533 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1536 if (PREDICT_FALSE(ip0->ttl == 1))
1538 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1539 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1540 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1542 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1546 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1548 /* Next configured feature, probably ip4-lookup */
1551 if (PREDICT_FALSE (proto0 == ~0))
1553 if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1555 next0 = SNAT_IN2OUT_NEXT_DROP;
1556 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1561 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1563 next0 = icmp_in2out_slow_path
1564 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1565 next0, now, thread_index, &s0);
1571 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1573 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1577 if (ip4_is_fragment (ip0))
1579 next0 = SNAT_IN2OUT_NEXT_REASS;
1584 key0.addr = ip0->src_address;
1585 key0.port = udp0->src_port;
1586 key0.protocol = proto0;
1587 key0.fib_index = rx_fib_index0;
1589 kv0.key = key0.as_u64;
1591 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1596 if (is_output_feature)
1598 if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1599 ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1604 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1605 ip0, proto0, rx_fib_index0, thread_index)))
1609 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1610 &s0, node, next0, thread_index);
1612 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1617 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1622 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1625 b0->flags |= VNET_BUFFER_F_IS_NATED;
1627 old_addr0 = ip0->src_address.as_u32;
1628 ip0->src_address = s0->out2in.addr;
1629 new_addr0 = ip0->src_address.as_u32;
1630 if (!is_output_feature)
1631 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1633 sum0 = ip0->checksum;
1634 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1636 src_address /* changed member */);
1637 ip0->checksum = ip_csum_fold (sum0);
1639 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1641 old_port0 = tcp0->src_port;
1642 tcp0->src_port = s0->out2in.port;
1643 new_port0 = tcp0->src_port;
1645 sum0 = tcp0->checksum;
1646 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1648 dst_address /* changed member */);
1649 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1650 ip4_header_t /* cheat */,
1651 length /* changed member */);
1652 tcp0->checksum = ip_csum_fold(sum0);
1656 old_port0 = udp0->src_port;
1657 udp0->src_port = s0->out2in.port;
1662 nat44_session_update_counters (s0, now,
1663 vlib_buffer_length_in_chain (vm, b0));
1664 /* Per-user LRU list maintenance */
1665 nat44_session_update_lru (sm, s0, thread_index);
1668 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1669 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1671 snat_in2out_trace_t *t =
1672 vlib_add_trace (vm, node, b0, sizeof (*t));
1673 t->is_slow_path = is_slow_path;
1674 t->sw_if_index = sw_if_index0;
1675 t->next_index = next0;
1676 t->session_index = ~0;
1678 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1681 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1683 /* verify speculative enqueue, maybe switch current next frame */
1684 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1685 to_next, n_left_to_next,
1689 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1692 vlib_node_increment_counter (vm, stats_node_index,
1693 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1695 return frame->n_vectors;
1699 snat_in2out_fast_path_fn (vlib_main_t * vm,
1700 vlib_node_runtime_t * node,
1701 vlib_frame_t * frame)
1703 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1706 VLIB_REGISTER_NODE (snat_in2out_node) = {
1707 .function = snat_in2out_fast_path_fn,
1708 .name = "nat44-in2out",
1709 .vector_size = sizeof (u32),
1710 .format_trace = format_snat_in2out_trace,
1711 .type = VLIB_NODE_TYPE_INTERNAL,
1713 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1714 .error_strings = snat_in2out_error_strings,
1716 .runtime_data_bytes = sizeof (snat_runtime_t),
1718 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1720 /* edit / add dispositions here */
1722 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1723 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1724 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1725 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1726 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1730 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1733 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1734 vlib_node_runtime_t * node,
1735 vlib_frame_t * frame)
1737 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1740 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1741 .function = snat_in2out_output_fast_path_fn,
1742 .name = "nat44-in2out-output",
1743 .vector_size = sizeof (u32),
1744 .format_trace = format_snat_in2out_trace,
1745 .type = VLIB_NODE_TYPE_INTERNAL,
1747 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1748 .error_strings = snat_in2out_error_strings,
1750 .runtime_data_bytes = sizeof (snat_runtime_t),
1752 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1754 /* edit / add dispositions here */
1756 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1757 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1758 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1759 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1760 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1764 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1765 snat_in2out_output_fast_path_fn);
1768 snat_in2out_slow_path_fn (vlib_main_t * vm,
1769 vlib_node_runtime_t * node,
1770 vlib_frame_t * frame)
1772 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
1775 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1776 .function = snat_in2out_slow_path_fn,
1777 .name = "nat44-in2out-slowpath",
1778 .vector_size = sizeof (u32),
1779 .format_trace = format_snat_in2out_trace,
1780 .type = VLIB_NODE_TYPE_INTERNAL,
1782 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1783 .error_strings = snat_in2out_error_strings,
1785 .runtime_data_bytes = sizeof (snat_runtime_t),
1787 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1789 /* edit / add dispositions here */
1791 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1792 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1793 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1794 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1795 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1799 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1800 snat_in2out_slow_path_fn);
1803 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1804 vlib_node_runtime_t * node,
1805 vlib_frame_t * frame)
1807 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
1810 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1811 .function = snat_in2out_output_slow_path_fn,
1812 .name = "nat44-in2out-output-slowpath",
1813 .vector_size = sizeof (u32),
1814 .format_trace = format_snat_in2out_trace,
1815 .type = VLIB_NODE_TYPE_INTERNAL,
1817 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1818 .error_strings = snat_in2out_error_strings,
1820 .runtime_data_bytes = sizeof (snat_runtime_t),
1822 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1824 /* edit / add dispositions here */
1826 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1827 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1828 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1829 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1830 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1834 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1835 snat_in2out_output_slow_path_fn);
1837 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
1840 nat44_hairpinning_fn_inline (vlib_main_t * vm,
1841 vlib_node_runtime_t * node,
1842 vlib_frame_t * frame,
1845 u32 n_left_from, * from, * to_next, stats_node_index;
1846 snat_in2out_next_t next_index;
1847 u32 pkts_processed = 0;
1848 snat_main_t * sm = &snat_main;
1849 vnet_feature_main_t *fm = &feature_main;
1850 u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1851 vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
1853 stats_node_index = is_ed ? nat44_ed_hairpinning_node.index :
1854 nat44_hairpinning_node.index;
1855 from = vlib_frame_vector_args (frame);
1856 n_left_from = frame->n_vectors;
1857 next_index = node->cached_next_index;
1859 while (n_left_from > 0)
1863 vlib_get_next_frame (vm, node, next_index,
1864 to_next, n_left_to_next);
1866 while (n_left_from > 0 && n_left_to_next > 0)
1873 udp_header_t * udp0;
1874 tcp_header_t * tcp0;
1876 /* speculatively enqueue b0 to the current next frame */
1882 n_left_to_next -= 1;
1884 b0 = vlib_get_buffer (vm, bi0);
1885 ip0 = vlib_buffer_get_current (b0);
1886 udp0 = ip4_next_header (ip0);
1887 tcp0 = (tcp_header_t *) udp0;
1889 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1891 vnet_get_config_data (&cm->config_main, &b0->current_config_index,
1894 if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
1895 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1897 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1899 /* verify speculative enqueue, maybe switch current next frame */
1900 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1901 to_next, n_left_to_next,
1905 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1908 vlib_node_increment_counter (vm, stats_node_index,
1909 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1911 return frame->n_vectors;
1915 nat44_hairpinning_fn (vlib_main_t * vm,
1916 vlib_node_runtime_t * node,
1917 vlib_frame_t * frame)
1919 return nat44_hairpinning_fn_inline (vm, node, frame, 0);
1922 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
1923 .function = nat44_hairpinning_fn,
1924 .name = "nat44-hairpinning",
1925 .vector_size = sizeof (u32),
1926 .type = VLIB_NODE_TYPE_INTERNAL,
1927 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1928 .error_strings = snat_in2out_error_strings,
1931 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1932 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1936 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
1937 nat44_hairpinning_fn);
1940 nat44_ed_hairpinning_fn (vlib_main_t * vm,
1941 vlib_node_runtime_t * node,
1942 vlib_frame_t * frame)
1944 return nat44_hairpinning_fn_inline (vm, node, frame, 1);
1947 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
1948 .function = nat44_ed_hairpinning_fn,
1949 .name = "nat44-ed-hairpinning",
1950 .vector_size = sizeof (u32),
1951 .type = VLIB_NODE_TYPE_INTERNAL,
1952 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1953 .error_strings = snat_in2out_error_strings,
1956 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1957 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1961 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpinning_node,
1962 nat44_ed_hairpinning_fn);
1965 nat44_reass_hairpinning (snat_main_t *sm,
1972 snat_session_key_t key0, sm0;
1973 snat_session_t * s0;
1974 clib_bihash_kv_8_8_t kv0, value0;
1976 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
1977 u16 new_dst_port0, old_dst_port0;
1978 udp_header_t * udp0;
1979 tcp_header_t * tcp0;
1981 key0.addr = ip0->dst_address;
1983 key0.protocol = proto0;
1984 key0.fib_index = sm->outside_fib_index;
1985 kv0.key = key0.as_u64;
1987 udp0 = ip4_next_header (ip0);
1989 /* Check if destination is static mappings */
1990 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1992 new_dst_addr0 = sm0.addr.as_u32;
1993 new_dst_port0 = sm0.port;
1994 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1996 /* or active sessions */
1999 if (sm->num_workers > 1)
2000 ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2002 ti = sm->num_workers;
2004 if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2007 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2008 new_dst_addr0 = s0->in2out.addr.as_u32;
2009 new_dst_port0 = s0->in2out.port;
2010 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2014 /* Destination is behind the same NAT, use internal address and port */
2017 old_dst_addr0 = ip0->dst_address.as_u32;
2018 ip0->dst_address.as_u32 = new_dst_addr0;
2019 sum0 = ip0->checksum;
2020 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2021 ip4_header_t, dst_address);
2022 ip0->checksum = ip_csum_fold (sum0);
2024 old_dst_port0 = dport;
2025 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2026 ip4_is_first_fragment (ip0)))
2028 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2030 tcp0 = ip4_next_header (ip0);
2031 tcp0->dst = new_dst_port0;
2032 sum0 = tcp0->checksum;
2033 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2034 ip4_header_t, dst_address);
2035 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2036 ip4_header_t /* cheat */, length);
2037 tcp0->checksum = ip_csum_fold(sum0);
2041 udp0->dst_port = new_dst_port0;
2047 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2049 tcp0 = ip4_next_header (ip0);
2050 sum0 = tcp0->checksum;
2051 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2052 ip4_header_t, dst_address);
2053 tcp0->checksum = ip_csum_fold(sum0);
2060 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2061 vlib_node_runtime_t * node,
2062 vlib_frame_t * frame)
2064 u32 n_left_from, *from, *to_next;
2065 snat_in2out_next_t next_index;
2066 u32 pkts_processed = 0;
2067 snat_main_t *sm = &snat_main;
2068 f64 now = vlib_time_now (vm);
2069 u32 thread_index = vm->thread_index;
2070 snat_main_per_thread_data_t *per_thread_data =
2071 &sm->per_thread_data[thread_index];
2072 u32 *fragments_to_drop = 0;
2073 u32 *fragments_to_loopback = 0;
2075 from = vlib_frame_vector_args (frame);
2076 n_left_from = frame->n_vectors;
2077 next_index = node->cached_next_index;
2079 while (n_left_from > 0)
2083 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2085 while (n_left_from > 0 && n_left_to_next > 0)
2087 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2092 nat_reass_ip4_t *reass0;
2093 udp_header_t * udp0;
2094 tcp_header_t * tcp0;
2095 snat_session_key_t key0;
2096 clib_bihash_kv_8_8_t kv0, value0;
2097 snat_session_t * s0 = 0;
2098 u16 old_port0, new_port0;
2101 /* speculatively enqueue b0 to the current next frame */
2107 n_left_to_next -= 1;
2109 b0 = vlib_get_buffer (vm, bi0);
2110 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2112 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2113 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2116 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2118 next0 = SNAT_IN2OUT_NEXT_DROP;
2119 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2123 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2124 udp0 = ip4_next_header (ip0);
2125 tcp0 = (tcp_header_t *) udp0;
2126 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2128 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2133 &fragments_to_drop);
2135 if (PREDICT_FALSE (!reass0))
2137 next0 = SNAT_IN2OUT_NEXT_DROP;
2138 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2139 nat_log_notice ("maximum reassemblies exceeded");
2143 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2145 key0.addr = ip0->src_address;
2146 key0.port = udp0->src_port;
2147 key0.protocol = proto0;
2148 key0.fib_index = rx_fib_index0;
2149 kv0.key = key0.as_u64;
2151 if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2153 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2154 ip0, proto0, rx_fib_index0, thread_index)))
2157 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2158 &s0, node, next0, thread_index);
2160 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2163 reass0->sess_index = s0 - per_thread_data->sessions;
2167 s0 = pool_elt_at_index (per_thread_data->sessions,
2169 reass0->sess_index = value0.value;
2171 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2175 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2177 if (nat_ip4_reass_add_fragment (reass0, bi0))
2179 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2180 nat_log_notice ("maximum fragments per reassembly exceeded");
2181 next0 = SNAT_IN2OUT_NEXT_DROP;
2187 s0 = pool_elt_at_index (per_thread_data->sessions,
2188 reass0->sess_index);
2191 old_addr0 = ip0->src_address.as_u32;
2192 ip0->src_address = s0->out2in.addr;
2193 new_addr0 = ip0->src_address.as_u32;
2194 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2196 sum0 = ip0->checksum;
2197 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2199 src_address /* changed member */);
2200 ip0->checksum = ip_csum_fold (sum0);
2202 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2204 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2206 old_port0 = tcp0->src_port;
2207 tcp0->src_port = s0->out2in.port;
2208 new_port0 = tcp0->src_port;
2210 sum0 = tcp0->checksum;
2211 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2213 dst_address /* changed member */);
2214 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2215 ip4_header_t /* cheat */,
2216 length /* changed member */);
2217 tcp0->checksum = ip_csum_fold(sum0);
2221 old_port0 = udp0->src_port;
2222 udp0->src_port = s0->out2in.port;
2228 nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2229 s0->ext_host_port, proto0);
2232 nat44_session_update_counters (s0, now,
2233 vlib_buffer_length_in_chain (vm, b0));
2234 /* Per-user LRU list maintenance */
2235 nat44_session_update_lru (sm, s0, thread_index);
2238 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2239 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2241 nat44_in2out_reass_trace_t *t =
2242 vlib_add_trace (vm, node, b0, sizeof (*t));
2243 t->cached = cached0;
2244 t->sw_if_index = sw_if_index0;
2245 t->next_index = next0;
2255 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2257 /* verify speculative enqueue, maybe switch current next frame */
2258 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2259 to_next, n_left_to_next,
2263 if (n_left_from == 0 && vec_len (fragments_to_loopback))
2265 from = vlib_frame_vector_args (frame);
2266 u32 len = vec_len (fragments_to_loopback);
2267 if (len <= VLIB_FRAME_SIZE)
2269 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2271 vec_reset_length (fragments_to_loopback);
2276 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2277 sizeof (u32) * VLIB_FRAME_SIZE);
2278 n_left_from = VLIB_FRAME_SIZE;
2279 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2284 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2287 vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2288 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2291 nat_send_all_to_node (vm, fragments_to_drop, node,
2292 &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2293 SNAT_IN2OUT_NEXT_DROP);
2295 vec_free (fragments_to_drop);
2296 vec_free (fragments_to_loopback);
2297 return frame->n_vectors;
2300 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2301 .function = nat44_in2out_reass_node_fn,
2302 .name = "nat44-in2out-reass",
2303 .vector_size = sizeof (u32),
2304 .format_trace = format_nat44_in2out_reass_trace,
2305 .type = VLIB_NODE_TYPE_INTERNAL,
2307 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2308 .error_strings = snat_in2out_error_strings,
2310 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2312 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2313 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2314 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2315 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2316 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2320 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2321 nat44_in2out_reass_node_fn);
2323 /*******************************/
2324 /*** endpoint-dependent mode ***/
2325 /*******************************/
2327 static_always_inline int
2328 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
2330 icmp46_header_t *icmp0;
2331 nat_ed_ses_key_t key0;
2332 icmp_echo_header_t *echo0, *inner_echo0 = 0;
2333 ip4_header_t *inner_ip0 = 0;
2334 void *l4_header = 0;
2335 icmp46_header_t *inner_icmp0;
2337 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2338 echo0 = (icmp_echo_header_t *)(icmp0+1);
2340 if (!icmp_is_error_message (icmp0))
2342 key0.proto = IP_PROTOCOL_ICMP;
2343 key0.l_addr = ip0->src_address;
2344 key0.r_addr = ip0->dst_address;
2345 key0.l_port = echo0->identifier;
2350 inner_ip0 = (ip4_header_t *)(echo0+1);
2351 l4_header = ip4_next_header (inner_ip0);
2352 key0.proto = inner_ip0->protocol;
2353 key0.r_addr = inner_ip0->src_address;
2354 key0.l_addr = inner_ip0->dst_address;
2355 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
2357 case SNAT_PROTOCOL_ICMP:
2358 inner_icmp0 = (icmp46_header_t*)l4_header;
2359 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2361 key0.l_port = inner_echo0->identifier;
2363 case SNAT_PROTOCOL_UDP:
2364 case SNAT_PROTOCOL_TCP:
2365 key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2366 key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
2369 return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
2377 slow_path_ed (snat_main_t *sm,
2380 clib_bihash_kv_16_8_t *kv,
2381 snat_session_t ** sessionp,
2382 vlib_node_runtime_t * node,
2388 snat_session_key_t key0, key1;
2389 u8 lb = 0, is_sm = 0;
2390 u32 address_index = ~0;
2391 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2392 nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
2393 u32 proto = ip_proto_to_snat_proto (key->proto);
2394 nat_outside_fib_t *outside_fib;
2395 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2396 fib_prefix_t pfx = {
2397 .fp_proto = FIB_PROTOCOL_IP4,
2400 .ip4.as_u32 = key->r_addr.as_u32,
2404 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
2406 b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2407 nat_ipfix_logging_max_sessions(sm->max_translations);
2408 nat_log_notice ("maximum sessions exceeded");
2409 return SNAT_IN2OUT_NEXT_DROP;
2412 key0.addr = key->l_addr;
2413 key0.port = key->l_port;
2414 key1.protocol = key0.protocol = proto;
2415 key0.fib_index = rx_fib_index;
2416 key1.fib_index = sm->outside_fib_index;
2417 /* First try to match static mapping by local address and port */
2418 if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb))
2420 /* Try to create dynamic translation */
2421 if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
2422 thread_index, &key1,
2424 sm->port_per_thread,
2425 tsm->snat_thread_index))
2427 nat_log_notice ("addresses exhausted");
2428 b->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2429 return SNAT_IN2OUT_NEXT_DROP;
2435 u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
2438 nat_log_warn ("create NAT user failed");
2439 return SNAT_IN2OUT_NEXT_DROP;
2442 s = nat_session_alloc_or_recycle (sm, u, thread_index);
2445 nat_log_warn ("create NAT session failed");
2446 return SNAT_IN2OUT_NEXT_DROP;
2449 user_session_increment (sm, u, is_sm);
2451 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2453 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
2454 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2455 s->outside_address_index = address_index;
2456 s->ext_host_addr = key->r_addr;
2457 s->ext_host_port = key->r_port;
2460 s->out2in.protocol = key0.protocol;
2462 switch (vec_len (sm->outside_fibs))
2465 s->out2in.fib_index = sm->outside_fib_index;
2468 s->out2in.fib_index = sm->outside_fibs[0].fib_index;
2471 vec_foreach (outside_fib, sm->outside_fibs)
2473 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2474 if (FIB_NODE_INDEX_INVALID != fei)
2476 if (fib_entry_get_resolving_interface (fei) != ~0)
2478 s->out2in.fib_index = outside_fib->fib_index;
2486 /* Add to lookup tables */
2487 kv->value = s - tsm->sessions;
2488 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, kv, 1))
2489 nat_log_notice ("in2out-ed key add failed");
2491 make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
2492 key1.port, key->r_port);
2493 kv->value = s - tsm->sessions;
2494 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, kv, 1))
2495 nat_log_notice ("out2in-ed key add failed");
2500 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
2501 s->out2in.addr.as_u32,
2505 s->in2out.fib_index);
2509 static_always_inline int
2510 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
2511 u32 sw_if_index, ip4_header_t * ip, u32 proto,
2512 u32 rx_fib_index, u32 thread_index)
2514 udp_header_t *udp = ip4_next_header (ip);
2515 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2516 clib_bihash_kv_16_8_t kv, value;
2517 snat_session_key_t key0, key1;
2519 make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
2520 sm->outside_fib_index, udp->dst_port, udp->src_port);
2522 /* NAT packet aimed at external address if */
2523 /* has active sessions */
2524 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2526 key0.addr = ip->dst_address;
2527 key0.port = udp->dst_port;
2528 key0.protocol = proto;
2529 key0.fib_index = sm->outside_fib_index;
2530 /* or is static mappings */
2531 if (!snat_static_mapping_match(sm, key0, &key1, 1, 0, 0, 0))
2537 if (sm->forwarding_enabled)
2540 return snat_not_translate_fast(sm, node, sw_if_index, ip, proto, rx_fib_index);
2543 static_always_inline int
2544 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
2545 u32 thread_index, f64 now,
2546 vlib_main_t * vm, vlib_buffer_t * b)
2548 nat_ed_ses_key_t key;
2549 clib_bihash_kv_16_8_t kv, value;
2551 snat_session_t *s = 0;
2552 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2554 if (!sm->forwarding_enabled)
2557 if (ip->protocol == IP_PROTOCOL_ICMP)
2559 key.as_u64[0] = key.as_u64[1] = 0;
2560 if (icmp_get_ed_key (ip, &key))
2563 kv.key[0] = key.as_u64[0];
2564 kv.key[1] = key.as_u64[1];
2566 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
2568 udp = ip4_next_header(ip);
2569 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
2570 udp->src_port, udp->dst_port);
2574 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
2578 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2580 s = pool_elt_at_index (tsm->sessions, value.value);
2581 if (is_fwd_bypass_session (s))
2583 if (ip->protocol == IP_PROTOCOL_TCP)
2585 tcp_header_t *tcp = ip4_next_header(ip);
2586 if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
2589 /* Per-user LRU list maintenance */
2590 nat44_session_update_lru (sm, s, thread_index);
2592 nat44_session_update_counters (s, now,
2593 vlib_buffer_length_in_chain (vm, b));
2603 static_always_inline int
2604 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
2605 u8 proto, u16 src_port, u16 dst_port,
2606 u32 thread_index, u32 sw_if_index)
2608 clib_bihash_kv_16_8_t kv, value;
2609 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2610 snat_interface_t *i;
2612 u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2615 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, fib_index,
2616 src_port, dst_port);
2617 if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2621 make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, fib_index,
2622 dst_port, src_port);
2623 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2625 s = pool_elt_at_index (tsm->sessions, value.value);
2626 if (is_fwd_bypass_session (s))
2630 pool_foreach (i, sm->output_feature_interfaces,
2632 if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
2642 icmp_match_in2out_ed(snat_main_t *sm, vlib_node_runtime_t *node,
2643 u32 thread_index, vlib_buffer_t *b, ip4_header_t *ip,
2644 u8 *p_proto, snat_session_key_t *p_value,
2645 u8 *p_dont_translate, void *d, void *e)
2647 icmp46_header_t *icmp;
2650 nat_ed_ses_key_t key;
2651 snat_session_t *s = 0;
2652 u8 dont_translate = 0;
2653 clib_bihash_kv_16_8_t kv, value;
2656 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2658 icmp = (icmp46_header_t *) ip4_next_header (ip);
2659 sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
2660 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2662 key.as_u64[0] = key.as_u64[1] = 0;
2663 err = icmp_get_ed_key (ip, &key);
2666 b->error = node->errors[err];
2667 next = SNAT_IN2OUT_NEXT_DROP;
2670 key.fib_index = rx_fib_index;
2672 kv.key[0] = key.as_u64[0];
2673 kv.key[1] = key.as_u64[1];
2675 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2677 if (vnet_buffer(b)->sw_if_index[VLIB_TX] != ~0)
2679 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(sm, ip,
2680 key.proto, key.l_port, key.r_port, thread_index, sw_if_index)))
2688 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node, sw_if_index,
2689 ip, SNAT_PROTOCOL_ICMP, rx_fib_index, thread_index)))
2696 if (PREDICT_FALSE(icmp_is_error_message (icmp)))
2698 b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2699 next = SNAT_IN2OUT_NEXT_DROP;
2703 next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
2706 if (PREDICT_FALSE (next == SNAT_IN2OUT_NEXT_DROP))
2711 if (PREDICT_FALSE(icmp->type != ICMP4_echo_request &&
2712 icmp->type != ICMP4_echo_reply &&
2713 !icmp_is_error_message (icmp)))
2715 b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2716 next = SNAT_IN2OUT_NEXT_DROP;
2720 s = pool_elt_at_index (tsm->sessions, value.value);
2723 *p_proto = ip_proto_to_snat_proto (key.proto);
2726 *p_value = s->out2in;
2727 *p_dont_translate = dont_translate;
2729 *(snat_session_t**)d = s;
2734 nat44_ed_hairpinning_unknown_proto (snat_main_t *sm,
2738 u32 old_addr, new_addr = 0, ti = 0;
2739 clib_bihash_kv_8_8_t kv, value;
2740 clib_bihash_kv_16_8_t s_kv, s_value;
2741 snat_static_mapping_t *m;
2744 snat_main_per_thread_data_t *tsm;
2746 if (sm->num_workers > 1)
2747 ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
2749 ti = sm->num_workers;
2750 tsm = &sm->per_thread_data[ti];
2752 old_addr = ip->dst_address.as_u32;
2753 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2754 sm->outside_fib_index, 0, 0);
2755 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2757 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
2758 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2761 m = pool_elt_at_index (sm->static_mappings, value.value);
2762 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2763 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
2764 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
2768 s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
2769 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2770 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2771 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
2774 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2775 ip->checksum = ip_csum_fold (sum);
2778 static snat_session_t *
2779 nat44_ed_in2out_unknown_proto (snat_main_t *sm,
2786 vlib_node_runtime_t * node)
2788 clib_bihash_kv_8_8_t kv, value;
2789 clib_bihash_kv_16_8_t s_kv, s_value;
2790 snat_static_mapping_t *m;
2791 u32 old_addr, new_addr = 0;
2794 dlist_elt_t *head, *elt;
2795 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2796 u32 elt_index, head_index, ses_index;
2798 u32 address_index = ~0, outside_fib_index = sm->outside_fib_index;
2801 nat_outside_fib_t *outside_fib;
2802 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2803 fib_prefix_t pfx = {
2804 .fp_proto = FIB_PROTOCOL_IP4,
2807 .ip4.as_u32 = ip->dst_address.as_u32,
2811 switch (vec_len (sm->outside_fibs))
2814 outside_fib_index = sm->outside_fib_index;
2817 outside_fib_index = sm->outside_fibs[0].fib_index;
2820 vec_foreach (outside_fib, sm->outside_fibs)
2822 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2823 if (FIB_NODE_INDEX_INVALID != fei)
2825 if (fib_entry_get_resolving_interface (fei) != ~0)
2827 outside_fib_index = outside_fib->fib_index;
2834 old_addr = ip->src_address.as_u32;
2836 make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
2837 rx_fib_index, 0, 0);
2839 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
2841 s = pool_elt_at_index (tsm->sessions, s_value.value);
2842 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
2846 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
2848 b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2849 nat_ipfix_logging_max_sessions(sm->max_translations);
2850 nat_log_notice ("maximum sessions exceeded");
2854 u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
2858 nat_log_warn ("create NAT user failed");
2862 make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
2864 /* Try to find static mapping first */
2865 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
2867 m = pool_elt_at_index (sm->static_mappings, value.value);
2868 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
2872 /* Fallback to 3-tuple key */
2875 /* Choose same out address as for TCP/UDP session to same destination */
2876 head_index = u->sessions_per_user_list_head_index;
2877 head = pool_elt_at_index (tsm->list_pool, head_index);
2878 elt_index = head->next;
2879 if (PREDICT_FALSE (elt_index == ~0))
2883 elt = pool_elt_at_index (tsm->list_pool, elt_index);
2884 ses_index = elt->value;
2887 while (ses_index != ~0)
2889 s = pool_elt_at_index (tsm->sessions, ses_index);
2890 elt_index = elt->next;
2891 elt = pool_elt_at_index (tsm->list_pool, elt_index);
2892 ses_index = elt->value;
2894 if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
2896 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
2897 address_index = s->outside_address_index;
2899 make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
2900 ip->protocol, outside_fib_index, 0, 0);
2901 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2908 for (i = 0; i < vec_len (sm->addresses); i++)
2910 make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
2911 ip->protocol, outside_fib_index, 0, 0);
2912 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2914 new_addr = ip->src_address.as_u32 =
2915 sm->addresses[i].addr.as_u32;
2924 s = nat_session_alloc_or_recycle (sm, u, thread_index);
2927 nat_log_warn ("create NAT session failed");
2931 s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
2932 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
2933 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2934 s->outside_address_index = address_index;
2935 s->out2in.addr.as_u32 = new_addr;
2936 s->out2in.fib_index = outside_fib_index;
2937 s->in2out.addr.as_u32 = old_addr;
2938 s->in2out.fib_index = rx_fib_index;
2939 s->in2out.port = s->out2in.port = ip->protocol;
2941 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2942 user_session_increment (sm, u, is_sm);
2944 /* Add to lookup tables */
2945 make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
2946 rx_fib_index, 0, 0);
2947 s_kv.value = s - tsm->sessions;
2948 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2949 nat_log_notice ("in2out key add failed");
2951 make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
2952 outside_fib_index, 0, 0);
2953 s_kv.value = s - tsm->sessions;
2954 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2955 nat_log_notice ("out2in key add failed");
2958 /* Update IP checksum */
2960 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
2961 ip->checksum = ip_csum_fold (sum);
2964 nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
2965 /* Per-user LRU list maintenance */
2966 nat44_session_update_lru (sm, s, thread_index);
2969 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2970 nat44_ed_hairpinning_unknown_proto(sm, b, ip);
2972 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2973 vnet_buffer(b)->sw_if_index[VLIB_TX] = outside_fib_index;
2979 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
2980 vlib_node_runtime_t * node,
2981 vlib_frame_t * frame, int is_slow_path,
2982 int is_output_feature)
2984 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2985 snat_in2out_next_t next_index;
2986 snat_main_t *sm = &snat_main;
2987 f64 now = vlib_time_now (vm);
2988 u32 thread_index = vm->thread_index;
2989 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2991 stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
2992 nat44_ed_in2out_node.index;
2994 from = vlib_frame_vector_args (frame);
2995 n_left_from = frame->n_vectors;
2996 next_index = node->cached_next_index;
2998 while (n_left_from > 0)
3002 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3004 while (n_left_from >= 4 && n_left_to_next >= 2)
3007 vlib_buffer_t *b0, *b1;
3008 u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3009 new_addr0, old_addr0;
3010 u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
3011 new_addr1, old_addr1;
3012 u16 old_port0, new_port0, old_port1, new_port1;
3013 ip4_header_t *ip0, *ip1;
3014 udp_header_t *udp0, *udp1;
3015 tcp_header_t *tcp0, *tcp1;
3016 icmp46_header_t *icmp0, *icmp1;
3017 snat_session_t *s0 = 0, *s1 = 0;
3018 clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
3019 ip_csum_t sum0, sum1;
3021 /* Prefetch next iteration. */
3023 vlib_buffer_t * p2, * p3;
3025 p2 = vlib_get_buffer (vm, from[2]);
3026 p3 = vlib_get_buffer (vm, from[3]);
3028 vlib_prefetch_buffer_header (p2, LOAD);
3029 vlib_prefetch_buffer_header (p3, LOAD);
3031 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3032 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3035 /* speculatively enqueue b0 and b1 to the current next frame */
3036 to_next[0] = bi0 = from[0];
3037 to_next[1] = bi1 = from[1];
3041 n_left_to_next -= 2;
3043 b0 = vlib_get_buffer (vm, bi0);
3044 b1 = vlib_get_buffer (vm, bi1);
3046 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3048 if (is_output_feature)
3049 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3051 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3054 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3055 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3058 if (PREDICT_FALSE(ip0->ttl == 1))
3060 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3061 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3062 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3064 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3068 udp0 = ip4_next_header (ip0);
3069 tcp0 = (tcp_header_t *) udp0;
3070 icmp0 = (icmp46_header_t *) udp0;
3071 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3075 if (PREDICT_FALSE (proto0 == ~0))
3077 s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3079 thread_index, now, vm,
3082 next0 = SNAT_IN2OUT_NEXT_DROP;
3086 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3088 next0 = icmp_in2out_slow_path
3089 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3090 next0, now, thread_index, &s0);
3096 if (is_output_feature)
3098 if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3099 sm, ip0, thread_index, now, vm, b0)))
3103 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3105 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3109 if (ip4_is_fragment (ip0))
3111 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3112 next0 = SNAT_IN2OUT_NEXT_DROP;
3117 make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3118 rx_fib_index0, udp0->src_port, udp0->dst_port);
3120 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3124 if (is_output_feature)
3126 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3127 sm, ip0, ip0->protocol, udp0->src_port,
3128 udp0->dst_port, thread_index, sw_if_index0)))
3133 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3134 sw_if_index0, ip0, proto0, rx_fib_index0,
3139 next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3140 next0, thread_index);
3142 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3147 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3153 s0 = pool_elt_at_index (tsm->sessions, value0.value);
3156 b0->flags |= VNET_BUFFER_F_IS_NATED;
3158 if (!is_output_feature)
3159 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3161 old_addr0 = ip0->src_address.as_u32;
3162 new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3163 sum0 = ip0->checksum;
3164 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3166 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3167 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3168 s0->ext_host_addr.as_u32, ip4_header_t,
3170 ip0->checksum = ip_csum_fold (sum0);
3172 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3174 old_port0 = tcp0->src_port;
3175 new_port0 = tcp0->src_port = s0->out2in.port;
3177 sum0 = tcp0->checksum;
3178 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3180 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3182 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3184 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3185 s0->ext_host_addr.as_u32,
3186 ip4_header_t, dst_address);
3187 sum0 = ip_csum_update (sum0, tcp0->dst_port,
3188 s0->ext_host_port, ip4_header_t,
3190 tcp0->dst_port = s0->ext_host_port;
3191 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3193 tcp0->checksum = ip_csum_fold(sum0);
3194 if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3199 udp0->src_port = s0->out2in.port;
3201 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3203 udp0->dst_port = s0->ext_host_port;
3204 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3209 nat44_session_update_counters (s0, now,
3210 vlib_buffer_length_in_chain (vm, b0));
3211 /* Per-user LRU list maintenance */
3212 nat44_session_update_lru (sm, s0, thread_index);
3215 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3216 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3218 snat_in2out_trace_t *t =
3219 vlib_add_trace (vm, node, b0, sizeof (*t));
3220 t->is_slow_path = is_slow_path;
3221 t->sw_if_index = sw_if_index0;
3222 t->next_index = next0;
3223 t->session_index = ~0;
3225 t->session_index = s0 - tsm->sessions;
3228 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3231 next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3233 if (is_output_feature)
3234 iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
3236 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
3239 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3240 rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3243 if (PREDICT_FALSE(ip1->ttl == 1))
3245 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3246 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3247 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3249 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3253 udp1 = ip4_next_header (ip1);
3254 tcp1 = (tcp_header_t *) udp1;
3255 icmp1 = (icmp46_header_t *) udp1;
3256 proto1 = ip_proto_to_snat_proto (ip1->protocol);
3260 if (PREDICT_FALSE (proto1 == ~0))
3262 s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
3264 thread_index, now, vm,
3267 next1 = SNAT_IN2OUT_NEXT_DROP;
3271 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
3273 next1 = icmp_in2out_slow_path
3274 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
3275 next1, now, thread_index, &s1);
3281 if (is_output_feature)
3283 if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3284 sm, ip1, thread_index, now, vm, b1)))
3288 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
3290 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3294 if (ip4_is_fragment (ip1))
3296 b1->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3297 next1 = SNAT_IN2OUT_NEXT_DROP;
3302 make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address, ip1->protocol,
3303 rx_fib_index1, udp1->src_port, udp1->dst_port);
3305 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
3309 if (is_output_feature)
3311 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3312 sm, ip1, ip1->protocol, udp1->src_port,
3313 udp1->dst_port, thread_index, sw_if_index1)))
3318 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3319 sw_if_index1, ip1, proto1, rx_fib_index1,
3324 next1 = slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
3325 next1, thread_index);
3327 if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
3332 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3338 s1 = pool_elt_at_index (tsm->sessions, value1.value);
3341 b1->flags |= VNET_BUFFER_F_IS_NATED;
3343 if (!is_output_feature)
3344 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
3346 old_addr1 = ip1->src_address.as_u32;
3347 new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
3348 sum1 = ip1->checksum;
3349 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3351 if (PREDICT_FALSE (is_twice_nat_session (s1)))
3352 sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3353 s1->ext_host_addr.as_u32, ip4_header_t,
3355 ip1->checksum = ip_csum_fold (sum1);
3357 if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
3359 old_port1 = tcp1->src_port;
3360 new_port1 = tcp1->src_port = s1->out2in.port;
3362 sum1 = tcp1->checksum;
3363 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3365 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
3367 if (PREDICT_FALSE (is_twice_nat_session (s1)))
3369 sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3370 s1->ext_host_addr.as_u32,
3371 ip4_header_t, dst_address);
3372 sum1 = ip_csum_update (sum1, tcp1->dst_port,
3373 s1->ext_host_port, ip4_header_t,
3375 tcp1->dst_port = s1->ext_host_port;
3376 ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3378 tcp1->checksum = ip_csum_fold(sum1);
3379 if (nat44_set_tcp_session_state_i2o (sm, s1, tcp1, thread_index))
3384 udp1->src_port = s1->out2in.port;
3386 if (PREDICT_FALSE (is_twice_nat_session (s1)))
3388 udp1->dst_port = s1->ext_host_port;
3389 ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3394 nat44_session_update_counters (s1, now,
3395 vlib_buffer_length_in_chain (vm, b1));
3396 /* Per-user LRU list maintenance */
3397 nat44_session_update_lru (sm, s1, thread_index);
3400 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3401 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3403 snat_in2out_trace_t *t =
3404 vlib_add_trace (vm, node, b1, sizeof (*t));
3405 t->is_slow_path = is_slow_path;
3406 t->sw_if_index = sw_if_index1;
3407 t->next_index = next1;
3408 t->session_index = ~0;
3410 t->session_index = s1 - tsm->sessions;
3413 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3415 /* verify speculative enqueues, maybe switch current next frame */
3416 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3417 to_next, n_left_to_next,
3418 bi0, bi1, next0, next1);
3421 while (n_left_from > 0 && n_left_to_next > 0)
3425 u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3426 new_addr0, old_addr0;
3427 u16 old_port0, new_port0;
3431 icmp46_header_t * icmp0;
3432 snat_session_t *s0 = 0;
3433 clib_bihash_kv_16_8_t kv0, value0;
3436 /* speculatively enqueue b0 to the current next frame */
3442 n_left_to_next -= 1;
3444 b0 = vlib_get_buffer (vm, bi0);
3445 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3447 if (is_output_feature)
3448 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3450 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3453 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3454 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3457 if (PREDICT_FALSE(ip0->ttl == 1))
3459 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3460 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3461 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3463 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3467 udp0 = ip4_next_header (ip0);
3468 tcp0 = (tcp_header_t *) udp0;
3469 icmp0 = (icmp46_header_t *) udp0;
3470 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3474 if (PREDICT_FALSE (proto0 == ~0))
3476 s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3478 thread_index, now, vm,
3481 next0 = SNAT_IN2OUT_NEXT_DROP;
3485 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3487 next0 = icmp_in2out_slow_path
3488 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3489 next0, now, thread_index, &s0);
3495 if (is_output_feature)
3497 if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3498 sm, ip0, thread_index, now, vm, b0)))
3502 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3504 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3508 if (ip4_is_fragment (ip0))
3510 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3511 next0 = SNAT_IN2OUT_NEXT_DROP;
3516 make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3517 rx_fib_index0, udp0->src_port, udp0->dst_port);
3519 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3523 if (is_output_feature)
3525 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3526 sm, ip0, ip0->protocol, udp0->src_port,
3527 udp0->dst_port, thread_index, sw_if_index0)))
3532 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3533 sw_if_index0, ip0, proto0, rx_fib_index0,
3538 next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3539 next0, thread_index);
3541 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3546 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3552 s0 = pool_elt_at_index (tsm->sessions, value0.value);
3555 b0->flags |= VNET_BUFFER_F_IS_NATED;
3557 if (!is_output_feature)
3558 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3560 old_addr0 = ip0->src_address.as_u32;
3561 new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3562 sum0 = ip0->checksum;
3563 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3565 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3566 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3567 s0->ext_host_addr.as_u32, ip4_header_t,
3569 ip0->checksum = ip_csum_fold (sum0);
3571 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3573 old_port0 = tcp0->src_port;
3574 new_port0 = tcp0->src_port = s0->out2in.port;
3576 sum0 = tcp0->checksum;
3577 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3579 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3581 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3583 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3584 s0->ext_host_addr.as_u32,
3585 ip4_header_t, dst_address);
3586 sum0 = ip_csum_update (sum0, tcp0->dst_port,
3587 s0->ext_host_port, ip4_header_t,
3589 tcp0->dst_port = s0->ext_host_port;
3590 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3592 tcp0->checksum = ip_csum_fold(sum0);
3593 if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3598 udp0->src_port = s0->out2in.port;
3600 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3602 udp0->dst_port = s0->ext_host_port;
3603 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3608 nat44_session_update_counters (s0, now,
3609 vlib_buffer_length_in_chain (vm, b0));
3610 /* Per-user LRU list maintenance */
3611 nat44_session_update_lru (sm, s0, thread_index);
3614 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3615 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3617 snat_in2out_trace_t *t =
3618 vlib_add_trace (vm, node, b0, sizeof (*t));
3619 t->is_slow_path = is_slow_path;
3620 t->sw_if_index = sw_if_index0;
3621 t->next_index = next0;
3622 t->session_index = ~0;
3624 t->session_index = s0 - tsm->sessions;
3627 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3629 /* verify speculative enqueue, maybe switch current next frame */
3630 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3631 to_next, n_left_to_next,
3635 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3638 vlib_node_increment_counter (vm, stats_node_index,
3639 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3641 return frame->n_vectors;
3645 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
3646 vlib_node_runtime_t * node,
3647 vlib_frame_t * frame)
3649 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
3652 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
3653 .function = nat44_ed_in2out_fast_path_fn,
3654 .name = "nat44-ed-in2out",
3655 .vector_size = sizeof (u32),
3656 .format_trace = format_snat_in2out_trace,
3657 .type = VLIB_NODE_TYPE_INTERNAL,
3659 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3660 .error_strings = snat_in2out_error_strings,
3662 .runtime_data_bytes = sizeof (snat_runtime_t),
3664 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3666 /* edit / add dispositions here */
3668 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3669 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3670 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3671 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3672 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3676 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node, nat44_ed_in2out_fast_path_fn);
3679 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
3680 vlib_node_runtime_t * node,
3681 vlib_frame_t * frame)
3683 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
3686 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
3687 .function = nat44_ed_in2out_output_fast_path_fn,
3688 .name = "nat44-ed-in2out-output",
3689 .vector_size = sizeof (u32),
3690 .format_trace = format_snat_in2out_trace,
3691 .type = VLIB_NODE_TYPE_INTERNAL,
3693 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3694 .error_strings = snat_in2out_error_strings,
3696 .runtime_data_bytes = sizeof (snat_runtime_t),
3698 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3700 /* edit / add dispositions here */
3702 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3703 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3704 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3705 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3706 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3710 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
3711 nat44_ed_in2out_output_fast_path_fn);
3714 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
3715 vlib_node_runtime_t * node,
3716 vlib_frame_t * frame)
3718 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
3721 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
3722 .function = nat44_ed_in2out_slow_path_fn,
3723 .name = "nat44-ed-in2out-slowpath",
3724 .vector_size = sizeof (u32),
3725 .format_trace = format_snat_in2out_trace,
3726 .type = VLIB_NODE_TYPE_INTERNAL,
3728 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3729 .error_strings = snat_in2out_error_strings,
3731 .runtime_data_bytes = sizeof (snat_runtime_t),
3733 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3735 /* edit / add dispositions here */
3737 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3738 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3739 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3740 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3741 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3745 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
3746 nat44_ed_in2out_slow_path_fn);
3749 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
3750 vlib_node_runtime_t * node,
3751 vlib_frame_t * frame)
3753 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
3756 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
3757 .function = nat44_ed_in2out_output_slow_path_fn,
3758 .name = "nat44-ed-in2out-output-slowpath",
3759 .vector_size = sizeof (u32),
3760 .format_trace = format_snat_in2out_trace,
3761 .type = VLIB_NODE_TYPE_INTERNAL,
3763 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3764 .error_strings = snat_in2out_error_strings,
3766 .runtime_data_bytes = sizeof (snat_runtime_t),
3768 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3770 /* edit / add dispositions here */
3772 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3773 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3774 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3775 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3776 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3780 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
3781 nat44_ed_in2out_output_slow_path_fn);
3783 /**************************/
3784 /*** deterministic mode ***/
3785 /**************************/
3787 snat_det_in2out_node_fn (vlib_main_t * vm,
3788 vlib_node_runtime_t * node,
3789 vlib_frame_t * frame)
3791 u32 n_left_from, * from, * to_next;
3792 snat_in2out_next_t next_index;
3793 u32 pkts_processed = 0;
3794 snat_main_t * sm = &snat_main;
3795 u32 now = (u32) vlib_time_now (vm);
3796 u32 thread_index = vm->thread_index;
3798 from = vlib_frame_vector_args (frame);
3799 n_left_from = frame->n_vectors;
3800 next_index = node->cached_next_index;
3802 while (n_left_from > 0)
3806 vlib_get_next_frame (vm, node, next_index,
3807 to_next, n_left_to_next);
3809 while (n_left_from >= 4 && n_left_to_next >= 2)
3812 vlib_buffer_t * b0, * b1;
3814 u32 sw_if_index0, sw_if_index1;
3815 ip4_header_t * ip0, * ip1;
3816 ip_csum_t sum0, sum1;
3817 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
3818 u16 old_port0, new_port0, lo_port0, i0;
3819 u16 old_port1, new_port1, lo_port1, i1;
3820 udp_header_t * udp0, * udp1;
3821 tcp_header_t * tcp0, * tcp1;
3823 snat_det_out_key_t key0, key1;
3824 snat_det_map_t * dm0, * dm1;
3825 snat_det_session_t * ses0 = 0, * ses1 = 0;
3826 u32 rx_fib_index0, rx_fib_index1;
3827 icmp46_header_t * icmp0, * icmp1;
3829 /* Prefetch next iteration. */
3831 vlib_buffer_t * p2, * p3;
3833 p2 = vlib_get_buffer (vm, from[2]);
3834 p3 = vlib_get_buffer (vm, from[3]);
3836 vlib_prefetch_buffer_header (p2, LOAD);
3837 vlib_prefetch_buffer_header (p3, LOAD);
3839 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3840 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3843 /* speculatively enqueue b0 and b1 to the current next frame */
3844 to_next[0] = bi0 = from[0];
3845 to_next[1] = bi1 = from[1];
3849 n_left_to_next -= 2;
3851 b0 = vlib_get_buffer (vm, bi0);
3852 b1 = vlib_get_buffer (vm, bi1);
3854 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3855 next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3857 ip0 = vlib_buffer_get_current (b0);
3858 udp0 = ip4_next_header (ip0);
3859 tcp0 = (tcp_header_t *) udp0;
3861 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3863 if (PREDICT_FALSE(ip0->ttl == 1))
3865 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3866 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3867 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3869 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3873 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3875 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3877 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3878 icmp0 = (icmp46_header_t *) udp0;
3880 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3881 rx_fib_index0, node, next0, thread_index,
3886 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3887 if (PREDICT_FALSE(!dm0))
3889 nat_log_info ("no match for internal host %U",
3890 format_ip4_address, &ip0->src_address);
3891 next0 = SNAT_IN2OUT_NEXT_DROP;
3892 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3896 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3898 key0.ext_host_addr = ip0->dst_address;
3899 key0.ext_host_port = tcp0->dst;
3901 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3902 if (PREDICT_FALSE(!ses0))
3904 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3906 key0.out_port = clib_host_to_net_u16 (lo_port0 +
3907 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3909 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3912 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3915 if (PREDICT_FALSE(!ses0))
3917 /* too many sessions for user, send ICMP error packet */
3919 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3920 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3921 ICMP4_destination_unreachable_destination_unreachable_host,
3923 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3928 new_port0 = ses0->out.out_port;
3930 old_addr0.as_u32 = ip0->src_address.as_u32;
3931 ip0->src_address.as_u32 = new_addr0.as_u32;
3932 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3934 sum0 = ip0->checksum;
3935 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3937 src_address /* changed member */);
3938 ip0->checksum = ip_csum_fold (sum0);
3940 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3942 if (tcp0->flags & TCP_FLAG_SYN)
3943 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3944 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3945 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3946 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3947 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3948 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3949 snat_det_ses_close(dm0, ses0);
3950 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3951 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3952 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3953 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3955 old_port0 = tcp0->src;
3956 tcp0->src = new_port0;
3958 sum0 = tcp0->checksum;
3959 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3961 dst_address /* changed member */);
3962 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3963 ip4_header_t /* cheat */,
3964 length /* changed member */);
3965 tcp0->checksum = ip_csum_fold(sum0);
3969 ses0->state = SNAT_SESSION_UDP_ACTIVE;
3970 old_port0 = udp0->src_port;
3971 udp0->src_port = new_port0;
3977 case SNAT_SESSION_UDP_ACTIVE:
3978 ses0->expire = now + sm->udp_timeout;
3980 case SNAT_SESSION_TCP_SYN_SENT:
3981 case SNAT_SESSION_TCP_FIN_WAIT:
3982 case SNAT_SESSION_TCP_CLOSE_WAIT:
3983 case SNAT_SESSION_TCP_LAST_ACK:
3984 ses0->expire = now + sm->tcp_transitory_timeout;
3986 case SNAT_SESSION_TCP_ESTABLISHED:
3987 ses0->expire = now + sm->tcp_established_timeout;
3992 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3993 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3995 snat_in2out_trace_t *t =
3996 vlib_add_trace (vm, node, b0, sizeof (*t));
3997 t->is_slow_path = 0;
3998 t->sw_if_index = sw_if_index0;
3999 t->next_index = next0;
4000 t->session_index = ~0;
4002 t->session_index = ses0 - dm0->sessions;
4005 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4007 ip1 = vlib_buffer_get_current (b1);
4008 udp1 = ip4_next_header (ip1);
4009 tcp1 = (tcp_header_t *) udp1;
4011 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
4013 if (PREDICT_FALSE(ip1->ttl == 1))
4015 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4016 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
4017 ICMP4_time_exceeded_ttl_exceeded_in_transit,
4019 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4023 proto1 = ip_proto_to_snat_proto (ip1->protocol);
4025 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
4027 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
4028 icmp1 = (icmp46_header_t *) udp1;
4030 next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
4031 rx_fib_index1, node, next1, thread_index,
4036 dm1 = snat_det_map_by_user(sm, &ip1->src_address);
4037 if (PREDICT_FALSE(!dm1))
4039 nat_log_info ("no match for internal host %U",
4040 format_ip4_address, &ip0->src_address);
4041 next1 = SNAT_IN2OUT_NEXT_DROP;
4042 b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4046 snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
4048 key1.ext_host_addr = ip1->dst_address;
4049 key1.ext_host_port = tcp1->dst;
4051 ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
4052 if (PREDICT_FALSE(!ses1))
4054 for (i1 = 0; i1 < dm1->ports_per_host; i1++)
4056 key1.out_port = clib_host_to_net_u16 (lo_port1 +
4057 ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
4059 if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
4062 ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
4065 if (PREDICT_FALSE(!ses1))
4067 /* too many sessions for user, send ICMP error packet */
4069 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4070 icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
4071 ICMP4_destination_unreachable_destination_unreachable_host,
4073 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4078 new_port1 = ses1->out.out_port;
4080 old_addr1.as_u32 = ip1->src_address.as_u32;
4081 ip1->src_address.as_u32 = new_addr1.as_u32;
4082 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4084 sum1 = ip1->checksum;
4085 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4087 src_address /* changed member */);
4088 ip1->checksum = ip_csum_fold (sum1);
4090 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
4092 if (tcp1->flags & TCP_FLAG_SYN)
4093 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
4094 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
4095 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4096 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
4097 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
4098 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
4099 snat_det_ses_close(dm1, ses1);
4100 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4101 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
4102 else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
4103 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4105 old_port1 = tcp1->src;
4106 tcp1->src = new_port1;
4108 sum1 = tcp1->checksum;
4109 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4111 dst_address /* changed member */);
4112 sum1 = ip_csum_update (sum1, old_port1, new_port1,
4113 ip4_header_t /* cheat */,
4114 length /* changed member */);
4115 tcp1->checksum = ip_csum_fold(sum1);
4119 ses1->state = SNAT_SESSION_UDP_ACTIVE;
4120 old_port1 = udp1->src_port;
4121 udp1->src_port = new_port1;
4127 case SNAT_SESSION_UDP_ACTIVE:
4128 ses1->expire = now + sm->udp_timeout;
4130 case SNAT_SESSION_TCP_SYN_SENT:
4131 case SNAT_SESSION_TCP_FIN_WAIT:
4132 case SNAT_SESSION_TCP_CLOSE_WAIT:
4133 case SNAT_SESSION_TCP_LAST_ACK:
4134 ses1->expire = now + sm->tcp_transitory_timeout;
4136 case SNAT_SESSION_TCP_ESTABLISHED:
4137 ses1->expire = now + sm->tcp_established_timeout;
4142 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4143 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
4145 snat_in2out_trace_t *t =
4146 vlib_add_trace (vm, node, b1, sizeof (*t));
4147 t->is_slow_path = 0;
4148 t->sw_if_index = sw_if_index1;
4149 t->next_index = next1;
4150 t->session_index = ~0;
4152 t->session_index = ses1 - dm1->sessions;
4155 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
4157 /* verify speculative enqueues, maybe switch current next frame */
4158 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
4159 to_next, n_left_to_next,
4160 bi0, bi1, next0, next1);
4163 while (n_left_from > 0 && n_left_to_next > 0)
4171 ip4_address_t new_addr0, old_addr0;
4172 u16 old_port0, new_port0, lo_port0, i0;
4173 udp_header_t * udp0;
4174 tcp_header_t * tcp0;
4176 snat_det_out_key_t key0;
4177 snat_det_map_t * dm0;
4178 snat_det_session_t * ses0 = 0;
4180 icmp46_header_t * icmp0;
4182 /* speculatively enqueue b0 to the current next frame */
4188 n_left_to_next -= 1;
4190 b0 = vlib_get_buffer (vm, bi0);
4191 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4193 ip0 = vlib_buffer_get_current (b0);
4194 udp0 = ip4_next_header (ip0);
4195 tcp0 = (tcp_header_t *) udp0;
4197 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4199 if (PREDICT_FALSE(ip0->ttl == 1))
4201 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4202 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4203 ICMP4_time_exceeded_ttl_exceeded_in_transit,
4205 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4209 proto0 = ip_proto_to_snat_proto (ip0->protocol);
4211 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4213 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4214 icmp0 = (icmp46_header_t *) udp0;
4216 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4217 rx_fib_index0, node, next0, thread_index,
4222 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4223 if (PREDICT_FALSE(!dm0))
4225 nat_log_info ("no match for internal host %U",
4226 format_ip4_address, &ip0->src_address);
4227 next0 = SNAT_IN2OUT_NEXT_DROP;
4228 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4232 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4234 key0.ext_host_addr = ip0->dst_address;
4235 key0.ext_host_port = tcp0->dst;
4237 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4238 if (PREDICT_FALSE(!ses0))
4240 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4242 key0.out_port = clib_host_to_net_u16 (lo_port0 +
4243 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4245 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4248 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4251 if (PREDICT_FALSE(!ses0))
4253 /* too many sessions for user, send ICMP error packet */
4255 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4256 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4257 ICMP4_destination_unreachable_destination_unreachable_host,
4259 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4264 new_port0 = ses0->out.out_port;
4266 old_addr0.as_u32 = ip0->src_address.as_u32;
4267 ip0->src_address.as_u32 = new_addr0.as_u32;
4268 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4270 sum0 = ip0->checksum;
4271 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4273 src_address /* changed member */);
4274 ip0->checksum = ip_csum_fold (sum0);
4276 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4278 if (tcp0->flags & TCP_FLAG_SYN)
4279 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4280 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4281 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4282 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4283 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4284 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4285 snat_det_ses_close(dm0, ses0);
4286 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4287 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4288 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4289 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4291 old_port0 = tcp0->src;
4292 tcp0->src = new_port0;
4294 sum0 = tcp0->checksum;
4295 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4297 dst_address /* changed member */);
4298 sum0 = ip_csum_update (sum0, old_port0, new_port0,
4299 ip4_header_t /* cheat */,
4300 length /* changed member */);
4301 tcp0->checksum = ip_csum_fold(sum0);
4305 ses0->state = SNAT_SESSION_UDP_ACTIVE;
4306 old_port0 = udp0->src_port;
4307 udp0->src_port = new_port0;
4313 case SNAT_SESSION_UDP_ACTIVE:
4314 ses0->expire = now + sm->udp_timeout;
4316 case SNAT_SESSION_TCP_SYN_SENT:
4317 case SNAT_SESSION_TCP_FIN_WAIT:
4318 case SNAT_SESSION_TCP_CLOSE_WAIT:
4319 case SNAT_SESSION_TCP_LAST_ACK:
4320 ses0->expire = now + sm->tcp_transitory_timeout;
4322 case SNAT_SESSION_TCP_ESTABLISHED:
4323 ses0->expire = now + sm->tcp_established_timeout;
4328 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4329 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4331 snat_in2out_trace_t *t =
4332 vlib_add_trace (vm, node, b0, sizeof (*t));
4333 t->is_slow_path = 0;
4334 t->sw_if_index = sw_if_index0;
4335 t->next_index = next0;
4336 t->session_index = ~0;
4338 t->session_index = ses0 - dm0->sessions;
4341 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4343 /* verify speculative enqueue, maybe switch current next frame */
4344 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4345 to_next, n_left_to_next,
4349 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4352 vlib_node_increment_counter (vm, snat_det_in2out_node.index,
4353 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4355 return frame->n_vectors;
4358 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
4359 .function = snat_det_in2out_node_fn,
4360 .name = "nat44-det-in2out",
4361 .vector_size = sizeof (u32),
4362 .format_trace = format_snat_in2out_trace,
4363 .type = VLIB_NODE_TYPE_INTERNAL,
4365 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4366 .error_strings = snat_in2out_error_strings,
4368 .runtime_data_bytes = sizeof (snat_runtime_t),
4372 /* edit / add dispositions here */
4374 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4375 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4376 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4380 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
4383 * Get address and port values to be used for ICMP packet translation
4384 * and create session if needed
4386 * @param[in,out] sm NAT main
4387 * @param[in,out] node NAT node runtime
4388 * @param[in] thread_index thread index
4389 * @param[in,out] b0 buffer containing packet to be translated
4390 * @param[out] p_proto protocol used for matching
4391 * @param[out] p_value address and port after NAT translation
4392 * @param[out] p_dont_translate if packet should not be translated
4393 * @param d optional parameter
4394 * @param e optional parameter
4396 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
4397 u32 thread_index, vlib_buffer_t *b0,
4398 ip4_header_t *ip0, u8 *p_proto,
4399 snat_session_key_t *p_value,
4400 u8 *p_dont_translate, void *d, void *e)
4402 icmp46_header_t *icmp0;
4406 snat_det_out_key_t key0;
4407 u8 dont_translate = 0;
4409 icmp_echo_header_t *echo0, *inner_echo0 = 0;
4410 ip4_header_t *inner_ip0;
4411 void *l4_header = 0;
4412 icmp46_header_t *inner_icmp0;
4413 snat_det_map_t * dm0 = 0;
4414 ip4_address_t new_addr0;
4416 snat_det_session_t * ses0 = 0;
4417 ip4_address_t in_addr;
4420 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
4421 echo0 = (icmp_echo_header_t *)(icmp0+1);
4422 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4423 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
4425 if (!icmp_is_error_message (icmp0))
4427 protocol = SNAT_PROTOCOL_ICMP;
4428 in_addr = ip0->src_address;
4429 in_port = echo0->identifier;
4433 inner_ip0 = (ip4_header_t *)(echo0+1);
4434 l4_header = ip4_next_header (inner_ip0);
4435 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
4436 in_addr = inner_ip0->dst_address;
4439 case SNAT_PROTOCOL_ICMP:
4440 inner_icmp0 = (icmp46_header_t*)l4_header;
4441 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
4442 in_port = inner_echo0->identifier;
4444 case SNAT_PROTOCOL_UDP:
4445 case SNAT_PROTOCOL_TCP:
4446 in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
4449 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
4450 next0 = SNAT_IN2OUT_NEXT_DROP;
4455 dm0 = snat_det_map_by_user(sm, &in_addr);
4456 if (PREDICT_FALSE(!dm0))
4458 nat_log_info ("no match for internal host %U",
4459 format_ip4_address, &in_addr);
4460 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4461 IP_PROTOCOL_ICMP, rx_fib_index0)))
4466 next0 = SNAT_IN2OUT_NEXT_DROP;
4467 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4471 snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
4473 key0.ext_host_addr = ip0->dst_address;
4474 key0.ext_host_port = 0;
4476 ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
4477 if (PREDICT_FALSE(!ses0))
4479 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4480 IP_PROTOCOL_ICMP, rx_fib_index0)))
4485 if (icmp0->type != ICMP4_echo_request)
4487 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4488 next0 = SNAT_IN2OUT_NEXT_DROP;
4491 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4493 key0.out_port = clib_host_to_net_u16 (lo_port0 +
4494 ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
4496 if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
4499 ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
4502 if (PREDICT_FALSE(!ses0))
4504 next0 = SNAT_IN2OUT_NEXT_DROP;
4505 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
4510 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
4511 !icmp_is_error_message (icmp0)))
4513 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4514 next0 = SNAT_IN2OUT_NEXT_DROP;
4518 u32 now = (u32) vlib_time_now (sm->vlib_main);
4520 ses0->state = SNAT_SESSION_ICMP_ACTIVE;
4521 ses0->expire = now + sm->icmp_timeout;
4524 *p_proto = protocol;
4527 p_value->addr = new_addr0;
4528 p_value->fib_index = sm->outside_fib_index;
4529 p_value->port = ses0->out.out_port;
4531 *p_dont_translate = dont_translate;
4533 *(snat_det_session_t**)d = ses0;
4535 *(snat_det_map_t**)e = dm0;
4539 /**********************/
4540 /*** worker handoff ***/
4541 /**********************/
4543 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
4544 vlib_node_runtime_t * node,
4545 vlib_frame_t * frame,
4548 snat_main_t *sm = &snat_main;
4549 vlib_thread_main_t *tm = vlib_get_thread_main ();
4550 u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
4551 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
4552 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
4554 vlib_frame_queue_elt_t *hf = 0;
4555 vlib_frame_queue_t *fq;
4556 vlib_frame_t *f = 0;
4558 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
4559 u32 next_worker_index = 0;
4560 u32 current_worker_index = ~0;
4561 u32 thread_index = vm->thread_index;
4564 vlib_frame_t *d = 0;
4566 ASSERT (vec_len (sm->workers));
4570 fq_index = sm->fq_in2out_output_index;
4571 to_node_index = sm->in2out_output_node_index;
4575 fq_index = sm->fq_in2out_index;
4576 to_node_index = sm->in2out_node_index;
4579 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
4581 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
4583 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
4584 tm->n_vlib_mains - 1,
4585 (vlib_frame_queue_t *) (~0));
4588 from = vlib_frame_vector_args (frame);
4589 n_left_from = frame->n_vectors;
4591 while (n_left_from > 0)
4604 b0 = vlib_get_buffer (vm, bi0);
4606 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
4607 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4609 ip0 = vlib_buffer_get_current (b0);
4611 next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
4613 if (PREDICT_FALSE (next_worker_index != thread_index))
4617 if (next_worker_index != current_worker_index)
4619 fq = is_vlib_frame_queue_congested (
4620 fq_index, next_worker_index, NAT_FQ_NELTS - 2,
4621 congested_handoff_queue_by_worker_index);
4625 /* if this is 1st frame */
4628 d = vlib_get_frame_to_node (vm, sm->error_node_index);
4629 to_next_drop = vlib_frame_vector_args (d);
4632 to_next_drop[0] = bi0;
4635 b0->error = node->errors[SNAT_IN2OUT_ERROR_FQ_CONGESTED];
4640 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4642 hf = vlib_get_worker_handoff_queue_elt (fq_index,
4644 handoff_queue_elt_by_worker_index);
4646 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
4647 to_next_worker = &hf->buffer_index[hf->n_vectors];
4648 current_worker_index = next_worker_index;
4651 /* enqueue to correct worker thread */
4652 to_next_worker[0] = bi0;
4654 n_left_to_next_worker--;
4656 if (n_left_to_next_worker == 0)
4658 hf->n_vectors = VLIB_FRAME_SIZE;
4659 vlib_put_frame_queue_elt (hf);
4660 current_worker_index = ~0;
4661 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
4668 /* if this is 1st frame */
4671 f = vlib_get_frame_to_node (vm, to_node_index);
4672 to_next = vlib_frame_vector_args (f);
4681 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
4682 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4684 snat_in2out_worker_handoff_trace_t *t =
4685 vlib_add_trace (vm, node, b0, sizeof (*t));
4686 t->next_worker_index = next_worker_index;
4687 t->do_handoff = do_handoff;
4692 vlib_put_frame_to_node (vm, to_node_index, f);
4695 vlib_put_frame_to_node (vm, sm->error_node_index, d);
4698 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4700 /* Ship frames to the worker nodes */
4701 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
4703 if (handoff_queue_elt_by_worker_index[i])
4705 hf = handoff_queue_elt_by_worker_index[i];
4707 * It works better to let the handoff node
4708 * rate-adapt, always ship the handoff queue element.
4710 if (1 || hf->n_vectors == hf->last_n_vectors)
4712 vlib_put_frame_queue_elt (hf);
4713 handoff_queue_elt_by_worker_index[i] = 0;
4716 hf->last_n_vectors = hf->n_vectors;
4718 congested_handoff_queue_by_worker_index[i] =
4719 (vlib_frame_queue_t *) (~0);
4722 current_worker_index = ~0;
4723 return frame->n_vectors;
4727 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
4728 vlib_node_runtime_t * node,
4729 vlib_frame_t * frame)
4731 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
4734 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
4735 .function = snat_in2out_worker_handoff_fn,
4736 .name = "nat44-in2out-worker-handoff",
4737 .vector_size = sizeof (u32),
4738 .format_trace = format_snat_in2out_worker_handoff_trace,
4739 .type = VLIB_NODE_TYPE_INTERNAL,
4741 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4742 .error_strings = snat_in2out_error_strings,
4751 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
4752 snat_in2out_worker_handoff_fn);
4755 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
4756 vlib_node_runtime_t * node,
4757 vlib_frame_t * frame)
4759 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
4762 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
4763 .function = snat_in2out_output_worker_handoff_fn,
4764 .name = "nat44-in2out-output-worker-handoff",
4765 .vector_size = sizeof (u32),
4766 .format_trace = format_snat_in2out_worker_handoff_trace,
4767 .type = VLIB_NODE_TYPE_INTERNAL,
4776 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
4777 snat_in2out_output_worker_handoff_fn);
4779 static_always_inline int
4780 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
4782 snat_address_t * ap;
4783 clib_bihash_kv_8_8_t kv, value;
4784 snat_session_key_t m_key;
4786 vec_foreach (ap, sm->addresses)
4788 if (ap->addr.as_u32 == dst_addr->as_u32)
4792 m_key.addr.as_u32 = dst_addr->as_u32;
4793 m_key.fib_index = 0;
4796 kv.key = m_key.as_u64;
4797 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4804 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
4805 vlib_node_runtime_t * node,
4806 vlib_frame_t * frame,
4809 u32 n_left_from, * from, * to_next, stats_node_index;
4810 snat_in2out_next_t next_index;
4811 u32 pkts_processed = 0;
4812 snat_main_t * sm = &snat_main;
4814 stats_node_index = is_ed ? nat44_ed_hairpin_dst_node.index :
4815 snat_hairpin_dst_node.index;
4817 from = vlib_frame_vector_args (frame);
4818 n_left_from = frame->n_vectors;
4819 next_index = node->cached_next_index;
4821 while (n_left_from > 0)
4825 vlib_get_next_frame (vm, node, next_index,
4826 to_next, n_left_to_next);
4828 while (n_left_from > 0 && n_left_to_next > 0)
4836 /* speculatively enqueue b0 to the current next frame */
4842 n_left_to_next -= 1;
4844 b0 = vlib_get_buffer (vm, bi0);
4845 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4846 ip0 = vlib_buffer_get_current (b0);
4848 proto0 = ip_proto_to_snat_proto (ip0->protocol);
4850 vnet_buffer (b0)->snat.flags = 0;
4851 if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
4853 if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
4855 udp_header_t * udp0 = ip4_next_header (ip0);
4856 tcp_header_t * tcp0 = (tcp_header_t *) udp0;
4858 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
4860 else if (proto0 == SNAT_PROTOCOL_ICMP)
4862 icmp46_header_t * icmp0 = ip4_next_header (ip0);
4864 snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
4869 nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
4871 nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
4874 vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
4877 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4879 /* verify speculative enqueue, maybe switch current next frame */
4880 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4881 to_next, n_left_to_next,
4885 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4888 vlib_node_increment_counter (vm, stats_node_index,
4889 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4891 return frame->n_vectors;
4895 snat_hairpin_dst_fn (vlib_main_t * vm,
4896 vlib_node_runtime_t * node,
4897 vlib_frame_t * frame)
4899 return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
4902 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
4903 .function = snat_hairpin_dst_fn,
4904 .name = "nat44-hairpin-dst",
4905 .vector_size = sizeof (u32),
4906 .type = VLIB_NODE_TYPE_INTERNAL,
4907 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4908 .error_strings = snat_in2out_error_strings,
4911 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4912 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4916 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
4917 snat_hairpin_dst_fn);
4920 nat44_ed_hairpin_dst_fn (vlib_main_t * vm,
4921 vlib_node_runtime_t * node,
4922 vlib_frame_t * frame)
4924 return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
4927 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
4928 .function = nat44_ed_hairpin_dst_fn,
4929 .name = "nat44-ed-hairpin-dst",
4930 .vector_size = sizeof (u32),
4931 .type = VLIB_NODE_TYPE_INTERNAL,
4932 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4933 .error_strings = snat_in2out_error_strings,
4936 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4937 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4941 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_dst_node,
4942 nat44_ed_hairpin_dst_fn);
4945 snat_hairpin_src_fn_inline (vlib_main_t * vm,
4946 vlib_node_runtime_t * node,
4947 vlib_frame_t * frame,
4950 u32 n_left_from, * from, * to_next, stats_node_index;
4951 snat_in2out_next_t next_index;
4952 u32 pkts_processed = 0;
4953 snat_main_t *sm = &snat_main;
4955 stats_node_index = is_ed ? nat44_ed_hairpin_src_node.index :
4956 snat_hairpin_src_node.index;
4958 from = vlib_frame_vector_args (frame);
4959 n_left_from = frame->n_vectors;
4960 next_index = node->cached_next_index;
4962 while (n_left_from > 0)
4966 vlib_get_next_frame (vm, node, next_index,
4967 to_next, n_left_to_next);
4969 while (n_left_from > 0 && n_left_to_next > 0)
4974 snat_interface_t *i;
4977 /* speculatively enqueue b0 to the current next frame */
4983 n_left_to_next -= 1;
4985 b0 = vlib_get_buffer (vm, bi0);
4986 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4987 next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
4989 pool_foreach (i, sm->output_feature_interfaces,
4991 /* Only packets from NAT inside interface */
4992 if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
4994 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
4995 SNAT_FLAG_HAIRPINNING))
4997 if (PREDICT_TRUE (sm->num_workers > 1))
4998 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
5000 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
5006 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5008 /* verify speculative enqueue, maybe switch current next frame */
5009 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5010 to_next, n_left_to_next,
5014 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5017 vlib_node_increment_counter (vm, stats_node_index,
5018 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5020 return frame->n_vectors;
5024 snat_hairpin_src_fn (vlib_main_t * vm,
5025 vlib_node_runtime_t * node,
5026 vlib_frame_t * frame)
5028 return snat_hairpin_src_fn_inline (vm, node, frame, 0);
5031 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
5032 .function = snat_hairpin_src_fn,
5033 .name = "nat44-hairpin-src",
5034 .vector_size = sizeof (u32),
5035 .type = VLIB_NODE_TYPE_INTERNAL,
5036 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5037 .error_strings = snat_in2out_error_strings,
5038 .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5040 [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5041 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
5042 [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5043 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5047 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
5048 snat_hairpin_src_fn);
5051 nat44_ed_hairpin_src_fn (vlib_main_t * vm,
5052 vlib_node_runtime_t * node,
5053 vlib_frame_t * frame)
5055 return snat_hairpin_src_fn_inline (vm, node, frame, 1);
5058 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
5059 .function = nat44_ed_hairpin_src_fn,
5060 .name = "nat44-ed-hairpin-src",
5061 .vector_size = sizeof (u32),
5062 .type = VLIB_NODE_TYPE_INTERNAL,
5063 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5064 .error_strings = snat_in2out_error_strings,
5065 .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5067 [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5068 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
5069 [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5070 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5074 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_src_node,
5075 nat44_ed_hairpin_src_fn);
5078 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
5079 vlib_node_runtime_t * node,
5080 vlib_frame_t * frame)
5082 u32 n_left_from, * from, * to_next;
5083 snat_in2out_next_t next_index;
5084 u32 pkts_processed = 0;
5085 snat_main_t * sm = &snat_main;
5086 u32 stats_node_index;
5088 stats_node_index = snat_in2out_fast_node.index;
5090 from = vlib_frame_vector_args (frame);
5091 n_left_from = frame->n_vectors;
5092 next_index = node->cached_next_index;
5094 while (n_left_from > 0)
5098 vlib_get_next_frame (vm, node, next_index,
5099 to_next, n_left_to_next);
5101 while (n_left_from > 0 && n_left_to_next > 0)
5109 u32 new_addr0, old_addr0;
5110 u16 old_port0, new_port0;
5111 udp_header_t * udp0;
5112 tcp_header_t * tcp0;
5113 icmp46_header_t * icmp0;
5114 snat_session_key_t key0, sm0;
5118 /* speculatively enqueue b0 to the current next frame */
5124 n_left_to_next -= 1;
5126 b0 = vlib_get_buffer (vm, bi0);
5127 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5129 ip0 = vlib_buffer_get_current (b0);
5130 udp0 = ip4_next_header (ip0);
5131 tcp0 = (tcp_header_t *) udp0;
5132 icmp0 = (icmp46_header_t *) udp0;
5134 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5135 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
5137 if (PREDICT_FALSE(ip0->ttl == 1))
5139 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
5140 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
5141 ICMP4_time_exceeded_ttl_exceeded_in_transit,
5143 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
5147 proto0 = ip_proto_to_snat_proto (ip0->protocol);
5149 if (PREDICT_FALSE (proto0 == ~0))
5152 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
5154 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
5155 rx_fib_index0, node, next0, ~0, 0, 0);
5159 key0.addr = ip0->src_address;
5160 key0.protocol = proto0;
5161 key0.port = udp0->src_port;
5162 key0.fib_index = rx_fib_index0;
5164 if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
5166 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
5167 next0= SNAT_IN2OUT_NEXT_DROP;
5171 new_addr0 = sm0.addr.as_u32;
5172 new_port0 = sm0.port;
5173 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
5174 old_addr0 = ip0->src_address.as_u32;
5175 ip0->src_address.as_u32 = new_addr0;
5177 sum0 = ip0->checksum;
5178 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5180 src_address /* changed member */);
5181 ip0->checksum = ip_csum_fold (sum0);
5183 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
5185 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5187 old_port0 = tcp0->src_port;
5188 tcp0->src_port = new_port0;
5190 sum0 = tcp0->checksum;
5191 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5193 dst_address /* changed member */);
5194 sum0 = ip_csum_update (sum0, old_port0, new_port0,
5195 ip4_header_t /* cheat */,
5196 length /* changed member */);
5197 tcp0->checksum = ip_csum_fold(sum0);
5201 old_port0 = udp0->src_port;
5202 udp0->src_port = new_port0;
5208 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5210 sum0 = tcp0->checksum;
5211 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5213 dst_address /* changed member */);
5214 tcp0->checksum = ip_csum_fold(sum0);
5219 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
5222 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
5223 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
5225 snat_in2out_trace_t *t =
5226 vlib_add_trace (vm, node, b0, sizeof (*t));
5227 t->sw_if_index = sw_if_index0;
5228 t->next_index = next0;
5231 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5233 /* verify speculative enqueue, maybe switch current next frame */
5234 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5235 to_next, n_left_to_next,
5239 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5242 vlib_node_increment_counter (vm, stats_node_index,
5243 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5245 return frame->n_vectors;
5249 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
5250 .function = snat_in2out_fast_static_map_fn,
5251 .name = "nat44-in2out-fast",
5252 .vector_size = sizeof (u32),
5253 .format_trace = format_snat_in2out_fast_trace,
5254 .type = VLIB_NODE_TYPE_INTERNAL,
5256 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5257 .error_strings = snat_in2out_error_strings,
5259 .runtime_data_bytes = sizeof (snat_runtime_t),
5261 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
5263 /* edit / add dispositions here */
5265 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5266 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5267 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
5268 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
5269 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
5273 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);