2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/fib/ip4_fib.h>
25 #include <nat/nat_ipfix_logging.h>
26 #include <nat/nat_det.h>
27 #include <nat/nat_reass.h>
29 #include <vppinfra/hash.h>
30 #include <vppinfra/error.h>
31 #include <vppinfra/elog.h>
38 } snat_in2out_trace_t;
41 u32 next_worker_index;
43 } snat_in2out_worker_handoff_trace_t;
45 /* packet trace format function */
46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
48 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
49 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
50 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
53 tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
55 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
56 t->sw_if_index, t->next_index, t->session_index);
61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
63 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
64 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
65 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
67 s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
68 t->sw_if_index, t->next_index);
73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
75 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
76 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
77 snat_in2out_worker_handoff_trace_t * t =
78 va_arg (*args, snat_in2out_worker_handoff_trace_t *);
81 m = t->do_handoff ? "next worker" : "same worker";
82 s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
91 } nat44_in2out_reass_trace_t;
93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
95 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
96 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
97 nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
99 s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
100 t->sw_if_index, t->next_index,
101 t->cached ? "cached" : "translated");
106 vlib_node_registration_t snat_in2out_node;
107 vlib_node_registration_t snat_in2out_slowpath_node;
108 vlib_node_registration_t snat_in2out_fast_node;
109 vlib_node_registration_t snat_in2out_worker_handoff_node;
110 vlib_node_registration_t snat_det_in2out_node;
111 vlib_node_registration_t snat_in2out_output_node;
112 vlib_node_registration_t snat_in2out_output_slowpath_node;
113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
114 vlib_node_registration_t snat_hairpin_dst_node;
115 vlib_node_registration_t snat_hairpin_src_node;
116 vlib_node_registration_t nat44_hairpinning_node;
117 vlib_node_registration_t nat44_in2out_reass_node;
120 #define foreach_snat_in2out_error \
121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
122 _(IN2OUT_PACKETS, "Good in2out packets processed") \
123 _(OUT_OF_PORTS, "Out of ports") \
124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \
125 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
126 _(NO_TRANSLATION, "No translation") \
127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
128 _(DROP_FRAGMENT, "Drop fragment") \
129 _(MAX_REASS, "Maximum reassemblies exceeded") \
130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
134 foreach_snat_in2out_error
137 } snat_in2out_error_t;
139 static char * snat_in2out_error_strings[] = {
140 #define _(sym,string) string,
141 foreach_snat_in2out_error
146 SNAT_IN2OUT_NEXT_LOOKUP,
147 SNAT_IN2OUT_NEXT_DROP,
148 SNAT_IN2OUT_NEXT_ICMP_ERROR,
149 SNAT_IN2OUT_NEXT_SLOW_PATH,
150 SNAT_IN2OUT_NEXT_REASS,
152 } snat_in2out_next_t;
155 SNAT_HAIRPIN_SRC_NEXT_DROP,
156 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
157 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
158 SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
159 SNAT_HAIRPIN_SRC_N_NEXT,
160 } snat_hairpin_next_t;
163 * @brief Check if packet should be translated
165 * Packets aimed at outside interface and external addresss with active session
166 * should be translated.
169 * @param rt NAT runtime data
170 * @param sw_if_index0 index of the inside interface
171 * @param ip0 IPv4 header
172 * @param proto0 NAT protocol
173 * @param rx_fib_index0 RX FIB index
175 * @returns 0 if packet should be translated otherwise 1
178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
179 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
182 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
184 .fp_proto = FIB_PROTOCOL_IP4,
187 .ip4.as_u32 = ip0->dst_address.as_u32,
191 /* Don't NAT packet aimed at the intfc address */
192 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
193 ip0->dst_address.as_u32)))
196 fei = fib_table_lookup (rx_fib_index0, &pfx);
197 if (FIB_NODE_INDEX_INVALID != fei)
199 u32 sw_if_index = fib_entry_get_resolving_interface (fei);
200 if (sw_if_index == ~0)
202 fei = fib_table_lookup (sm->outside_fib_index, &pfx);
203 if (FIB_NODE_INDEX_INVALID != fei)
204 sw_if_index = fib_entry_get_resolving_interface (fei);
207 pool_foreach (i, sm->interfaces,
209 /* NAT packet aimed at outside interface */
210 if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
219 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
220 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
221 u32 rx_fib_index0, u32 thread_index)
223 udp_header_t * udp0 = ip4_next_header (ip0);
224 snat_session_key_t key0, sm0;
225 clib_bihash_kv_8_8_t kv0, value0;
227 key0.addr = ip0->dst_address;
228 key0.port = udp0->dst_port;
229 key0.protocol = proto0;
230 key0.fib_index = sm->outside_fib_index;
231 kv0.key = key0.as_u64;
233 /* NAT packet aimed at external address if */
234 /* has active sessions */
235 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
238 /* or is static mappings */
239 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
245 return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
249 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
252 snat_session_key_t * key0,
253 snat_session_t ** sessionp,
254 vlib_node_runtime_t * node,
260 clib_bihash_kv_8_8_t kv0;
261 snat_session_key_t key1;
262 u32 address_index = ~0;
263 u32 outside_fib_index;
265 udp_header_t * udp0 = ip4_next_header (ip0);
267 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
269 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
270 return SNAT_IN2OUT_NEXT_DROP;
273 p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
276 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
277 return SNAT_IN2OUT_NEXT_DROP;
279 outside_fib_index = p[0];
281 key1.protocol = key0->protocol;
283 u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
287 clib_warning ("create NAT user failed");
288 return SNAT_IN2OUT_NEXT_DROP;
291 s = nat_session_alloc_or_recycle (sm, u, thread_index);
294 clib_warning ("create NAT session failed");
295 return SNAT_IN2OUT_NEXT_DROP;
298 /* First try to match static mapping by local address and port */
299 if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0))
301 /* Try to create dynamic translation */
302 if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
306 sm->per_thread_data[thread_index].snat_thread_index))
308 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
309 return SNAT_IN2OUT_NEXT_DROP;
315 u->nstaticsessions++;
316 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
319 s->outside_address_index = address_index;
322 s->out2in.protocol = key0->protocol;
323 s->out2in.fib_index = outside_fib_index;
324 s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
325 s->ext_host_port = udp0->dst_port;
328 /* Add to translation hashes */
329 kv0.key = s->in2out.as_u64;
330 kv0.value = s - sm->per_thread_data[thread_index].sessions;
331 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
333 clib_warning ("in2out key add failed");
335 kv0.key = s->out2in.as_u64;
336 kv0.value = s - sm->per_thread_data[thread_index].sessions;
338 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
340 clib_warning ("out2in key add failed");
343 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
344 s->out2in.addr.as_u32,
348 s->in2out.fib_index);
353 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
354 snat_session_key_t *p_key0)
356 icmp46_header_t *icmp0;
357 snat_session_key_t key0;
358 icmp_echo_header_t *echo0, *inner_echo0 = 0;
359 ip4_header_t *inner_ip0 = 0;
361 icmp46_header_t *inner_icmp0;
363 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
364 echo0 = (icmp_echo_header_t *)(icmp0+1);
366 if (!icmp_is_error_message (icmp0))
368 key0.protocol = SNAT_PROTOCOL_ICMP;
369 key0.addr = ip0->src_address;
370 key0.port = echo0->identifier;
374 inner_ip0 = (ip4_header_t *)(echo0+1);
375 l4_header = ip4_next_header (inner_ip0);
376 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
377 key0.addr = inner_ip0->dst_address;
378 switch (key0.protocol)
380 case SNAT_PROTOCOL_ICMP:
381 inner_icmp0 = (icmp46_header_t*)l4_header;
382 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
383 key0.port = inner_echo0->identifier;
385 case SNAT_PROTOCOL_UDP:
386 case SNAT_PROTOCOL_TCP:
387 key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
390 return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
394 return -1; /* success */
398 * Get address and port values to be used for ICMP packet translation
399 * and create session if needed
401 * @param[in,out] sm NAT main
402 * @param[in,out] node NAT node runtime
403 * @param[in] thread_index thread index
404 * @param[in,out] b0 buffer containing packet to be translated
405 * @param[out] p_proto protocol used for matching
406 * @param[out] p_value address and port after NAT translation
407 * @param[out] p_dont_translate if packet should not be translated
408 * @param d optional parameter
409 * @param e optional parameter
411 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
412 u32 thread_index, vlib_buffer_t *b0,
413 ip4_header_t *ip0, u8 *p_proto,
414 snat_session_key_t *p_value,
415 u8 *p_dont_translate, void *d, void *e)
417 icmp46_header_t *icmp0;
420 snat_session_key_t key0;
421 snat_session_t *s0 = 0;
422 u8 dont_translate = 0;
423 clib_bihash_kv_8_8_t kv0, value0;
427 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
428 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
429 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
431 err = icmp_get_key (ip0, &key0);
434 b0->error = node->errors[err];
435 next0 = SNAT_IN2OUT_NEXT_DROP;
438 key0.fib_index = rx_fib_index0;
440 kv0.key = key0.as_u64;
442 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
445 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0, ip0,
446 IP_PROTOCOL_ICMP, rx_fib_index0, thread_index) &&
447 vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0))
453 if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
455 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
456 next0 = SNAT_IN2OUT_NEXT_DROP;
460 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
461 &s0, node, next0, thread_index);
463 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
468 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
469 icmp0->type != ICMP4_echo_reply &&
470 !icmp_is_error_message (icmp0)))
472 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
473 next0 = SNAT_IN2OUT_NEXT_DROP;
477 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
482 *p_proto = key0.protocol;
484 *p_value = s0->out2in;
485 *p_dont_translate = dont_translate;
487 *(snat_session_t**)d = s0;
492 * Get address and port values to be used for ICMP packet translation
494 * @param[in] sm NAT main
495 * @param[in,out] node NAT node runtime
496 * @param[in] thread_index thread index
497 * @param[in,out] b0 buffer containing packet to be translated
498 * @param[out] p_proto protocol used for matching
499 * @param[out] p_value address and port after NAT translation
500 * @param[out] p_dont_translate if packet should not be translated
501 * @param d optional parameter
502 * @param e optional parameter
504 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
505 u32 thread_index, vlib_buffer_t *b0,
506 ip4_header_t *ip0, u8 *p_proto,
507 snat_session_key_t *p_value,
508 u8 *p_dont_translate, void *d, void *e)
510 icmp46_header_t *icmp0;
513 snat_session_key_t key0;
514 snat_session_key_t sm0;
515 u8 dont_translate = 0;
520 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
521 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
522 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
524 err = icmp_get_key (ip0, &key0);
527 b0->error = node->errors[err];
528 next0 = SNAT_IN2OUT_NEXT_DROP;
531 key0.fib_index = rx_fib_index0;
533 if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0))
535 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
536 IP_PROTOCOL_ICMP, rx_fib_index0)))
542 if (icmp_is_error_message (icmp0))
544 next0 = SNAT_IN2OUT_NEXT_DROP;
548 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
549 next0 = SNAT_IN2OUT_NEXT_DROP;
553 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
554 (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
555 !icmp_is_error_message (icmp0)))
557 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
558 next0 = SNAT_IN2OUT_NEXT_DROP;
565 *p_proto = key0.protocol;
566 *p_dont_translate = dont_translate;
570 static inline u32 icmp_in2out (snat_main_t *sm,
573 icmp46_header_t * icmp0,
576 vlib_node_runtime_t * node,
582 snat_session_key_t sm0;
584 icmp_echo_header_t *echo0, *inner_echo0 = 0;
585 ip4_header_t *inner_ip0;
587 icmp46_header_t *inner_icmp0;
589 u32 new_addr0, old_addr0;
590 u16 old_id0, new_id0;
595 echo0 = (icmp_echo_header_t *)(icmp0+1);
597 next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
598 &protocol, &sm0, &dont_translate, d, e);
601 if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
604 sum0 = ip_incremental_checksum (0, icmp0,
605 ntohs(ip0->length) - ip4_header_bytes (ip0));
606 checksum0 = ~ip_csum_fold (sum0);
607 if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
609 next0 = SNAT_IN2OUT_NEXT_DROP;
613 old_addr0 = ip0->src_address.as_u32;
614 new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
615 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
616 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
618 sum0 = ip0->checksum;
619 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
620 src_address /* changed member */);
621 ip0->checksum = ip_csum_fold (sum0);
623 if (!icmp_is_error_message (icmp0))
626 if (PREDICT_FALSE(new_id0 != echo0->identifier))
628 old_id0 = echo0->identifier;
630 echo0->identifier = new_id0;
632 sum0 = icmp0->checksum;
633 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
635 icmp0->checksum = ip_csum_fold (sum0);
640 inner_ip0 = (ip4_header_t *)(echo0+1);
641 l4_header = ip4_next_header (inner_ip0);
643 if (!ip4_header_checksum_is_valid (inner_ip0))
645 next0 = SNAT_IN2OUT_NEXT_DROP;
649 old_addr0 = inner_ip0->dst_address.as_u32;
650 inner_ip0->dst_address = sm0.addr;
651 new_addr0 = inner_ip0->dst_address.as_u32;
653 sum0 = icmp0->checksum;
654 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
655 dst_address /* changed member */);
656 icmp0->checksum = ip_csum_fold (sum0);
660 case SNAT_PROTOCOL_ICMP:
661 inner_icmp0 = (icmp46_header_t*)l4_header;
662 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
664 old_id0 = inner_echo0->identifier;
666 inner_echo0->identifier = new_id0;
668 sum0 = icmp0->checksum;
669 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
671 icmp0->checksum = ip_csum_fold (sum0);
673 case SNAT_PROTOCOL_UDP:
674 case SNAT_PROTOCOL_TCP:
675 old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
677 ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
679 sum0 = icmp0->checksum;
680 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
682 icmp0->checksum = ip_csum_fold (sum0);
696 * Hairpinning allows two endpoints on the internal side of the NAT to
697 * communicate even if they only use each other's external IP addresses
700 * @param sm NAT main.
701 * @param b0 Vlib buffer.
702 * @param ip0 IP header.
703 * @param udp0 UDP header.
704 * @param tcp0 TCP header.
705 * @param proto0 NAT protocol.
708 snat_hairpinning (snat_main_t *sm,
715 snat_session_key_t key0, sm0;
717 clib_bihash_kv_8_8_t kv0, value0;
719 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
720 u16 new_dst_port0, old_dst_port0;
722 key0.addr = ip0->dst_address;
723 key0.port = udp0->dst_port;
724 key0.protocol = proto0;
725 key0.fib_index = sm->outside_fib_index;
726 kv0.key = key0.as_u64;
728 /* Check if destination is static mappings */
729 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
731 new_dst_addr0 = sm0.addr.as_u32;
732 new_dst_port0 = sm0.port;
733 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
735 /* or active session */
738 if (sm->num_workers > 1)
739 ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
741 ti = sm->num_workers;
743 if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
747 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
748 new_dst_addr0 = s0->in2out.addr.as_u32;
749 new_dst_port0 = s0->in2out.port;
750 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
754 /* Destination is behind the same NAT, use internal address and port */
757 old_dst_addr0 = ip0->dst_address.as_u32;
758 ip0->dst_address.as_u32 = new_dst_addr0;
759 sum0 = ip0->checksum;
760 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
761 ip4_header_t, dst_address);
762 ip0->checksum = ip_csum_fold (sum0);
764 old_dst_port0 = tcp0->dst;
765 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
767 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
769 tcp0->dst = new_dst_port0;
770 sum0 = tcp0->checksum;
771 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
772 ip4_header_t, dst_address);
773 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
774 ip4_header_t /* cheat */, length);
775 tcp0->checksum = ip_csum_fold(sum0);
779 udp0->dst_port = new_dst_port0;
785 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
787 sum0 = tcp0->checksum;
788 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
789 ip4_header_t, dst_address);
790 tcp0->checksum = ip_csum_fold(sum0);
799 snat_icmp_hairpinning (snat_main_t *sm,
802 icmp46_header_t * icmp0)
804 snat_session_key_t key0, sm0;
805 clib_bihash_kv_8_8_t kv0, value0;
806 u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
810 if (!icmp_is_error_message (icmp0))
812 icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
813 u16 icmp_id0 = echo0->identifier;
814 key0.addr = ip0->dst_address;
815 key0.port = icmp_id0;
816 key0.protocol = SNAT_PROTOCOL_ICMP;
817 key0.fib_index = sm->outside_fib_index;
818 kv0.key = key0.as_u64;
820 if (sm->num_workers > 1)
821 ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
823 ti = sm->num_workers;
825 /* Check if destination is in active sessions */
826 if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
829 /* or static mappings */
830 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
832 new_dst_addr0 = sm0.addr.as_u32;
833 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
840 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
841 new_dst_addr0 = s0->in2out.addr.as_u32;
842 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
843 echo0->identifier = s0->in2out.port;
844 sum0 = icmp0->checksum;
845 sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
846 icmp_echo_header_t, identifier);
847 icmp0->checksum = ip_csum_fold (sum0);
850 /* Destination is behind the same NAT, use internal address and port */
853 old_dst_addr0 = ip0->dst_address.as_u32;
854 ip0->dst_address.as_u32 = new_dst_addr0;
855 sum0 = ip0->checksum;
856 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
857 ip4_header_t, dst_address);
858 ip0->checksum = ip_csum_fold (sum0);
864 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
867 icmp46_header_t * icmp0,
870 vlib_node_runtime_t * node,
874 snat_session_t ** p_s0)
876 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
877 next0, thread_index, p_s0, 0);
878 snat_session_t * s0 = *p_s0;
879 if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
882 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
883 snat_icmp_hairpinning(sm, b0, ip0, icmp0);
885 s0->last_heard = now;
887 s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
888 /* Per-user LRU list maintenance */
889 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
891 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
892 s0->per_user_list_head_index,
898 snat_hairpinning_unknown_proto (snat_main_t *sm,
902 u32 old_addr, new_addr = 0, ti = 0;
903 clib_bihash_kv_8_8_t kv, value;
904 clib_bihash_kv_16_8_t s_kv, s_value;
905 nat_ed_ses_key_t key;
906 snat_session_key_t m_key;
907 snat_static_mapping_t *m;
911 old_addr = ip->dst_address.as_u32;
912 key.l_addr.as_u32 = ip->dst_address.as_u32;
913 key.r_addr.as_u32 = ip->src_address.as_u32;
914 key.fib_index = sm->outside_fib_index;
915 key.proto = ip->protocol;
918 s_kv.key[0] = key.as_u64[0];
919 s_kv.key[1] = key.as_u64[1];
920 if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
922 m_key.addr = ip->dst_address;
923 m_key.fib_index = sm->outside_fib_index;
926 kv.key = m_key.as_u64;
927 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
930 m = pool_elt_at_index (sm->static_mappings, value.value);
931 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
932 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
933 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
937 if (sm->num_workers > 1)
938 ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
940 ti = sm->num_workers;
942 s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
943 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
944 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
945 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
948 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
949 ip->checksum = ip_csum_fold (sum);
952 static snat_session_t *
953 snat_in2out_unknown_proto (snat_main_t *sm,
960 vlib_node_runtime_t * node)
962 clib_bihash_kv_8_8_t kv, value;
963 clib_bihash_kv_16_8_t s_kv, s_value;
964 snat_static_mapping_t *m;
965 snat_session_key_t m_key;
966 u32 old_addr, new_addr = 0;
969 dlist_elt_t *head, *elt;
970 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
971 u32 elt_index, head_index, ses_index;
973 nat_ed_ses_key_t key;
974 u32 address_index = ~0;
978 old_addr = ip->src_address.as_u32;
980 key.l_addr = ip->src_address;
981 key.r_addr = ip->dst_address;
982 key.fib_index = rx_fib_index;
983 key.proto = ip->protocol;
986 s_kv.key[0] = key.as_u64[0];
987 s_kv.key[1] = key.as_u64[1];
989 if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
991 s = pool_elt_at_index (tsm->sessions, s_value.value);
992 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
996 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
998 b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1002 u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1006 clib_warning ("create NAT user failed");
1010 m_key.addr = ip->src_address;
1013 m_key.fib_index = rx_fib_index;
1014 kv.key = m_key.as_u64;
1016 /* Try to find static mapping first */
1017 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1019 m = pool_elt_at_index (sm->static_mappings, value.value);
1020 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1024 /* Fallback to 3-tuple key */
1027 /* Choose same out address as for TCP/UDP session to same destination */
1028 if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1030 head_index = u->sessions_per_user_list_head_index;
1031 head = pool_elt_at_index (tsm->list_pool, head_index);
1032 elt_index = head->next;
1033 elt = pool_elt_at_index (tsm->list_pool, elt_index);
1034 ses_index = elt->value;
1035 while (ses_index != ~0)
1037 s = pool_elt_at_index (tsm->sessions, ses_index);
1038 elt_index = elt->next;
1039 elt = pool_elt_at_index (tsm->list_pool, elt_index);
1040 ses_index = elt->value;
1042 if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1044 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1045 address_index = s->outside_address_index;
1047 key.fib_index = sm->outside_fib_index;
1048 key.l_addr.as_u32 = new_addr;
1049 s_kv.key[0] = key.as_u64[0];
1050 s_kv.key[1] = key.as_u64[1];
1051 if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1058 key.fib_index = sm->outside_fib_index;
1059 for (i = 0; i < vec_len (sm->addresses); i++)
1061 key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1062 s_kv.key[0] = key.as_u64[0];
1063 s_kv.key[1] = key.as_u64[1];
1064 if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1066 new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1075 s = nat_session_alloc_or_recycle (sm, u, thread_index);
1078 clib_warning ("create NAT session failed");
1082 s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1083 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1084 s->outside_address_index = address_index;
1085 s->out2in.addr.as_u32 = new_addr;
1086 s->out2in.fib_index = sm->outside_fib_index;
1087 s->in2out.addr.as_u32 = old_addr;
1088 s->in2out.fib_index = rx_fib_index;
1089 s->in2out.port = s->out2in.port = ip->protocol;
1092 u->nstaticsessions++;
1093 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1100 /* Add to lookup tables */
1101 key.l_addr.as_u32 = old_addr;
1102 key.r_addr = ip->dst_address;
1103 key.proto = ip->protocol;
1104 key.fib_index = rx_fib_index;
1105 s_kv.key[0] = key.as_u64[0];
1106 s_kv.key[1] = key.as_u64[1];
1107 s_kv.value = s - tsm->sessions;
1108 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1109 clib_warning ("in2out key add failed");
1111 key.l_addr.as_u32 = new_addr;
1112 key.fib_index = sm->outside_fib_index;
1113 s_kv.key[0] = key.as_u64[0];
1114 s_kv.key[1] = key.as_u64[1];
1115 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1116 clib_warning ("out2in key add failed");
1119 /* Update IP checksum */
1121 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1122 ip->checksum = ip_csum_fold (sum);
1125 s->last_heard = now;
1127 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1128 /* Per-user LRU list maintenance */
1129 clib_dlist_remove (tsm->list_pool, s->per_user_index);
1130 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1134 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1135 snat_hairpinning_unknown_proto(sm, b, ip);
1137 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1138 vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1143 static snat_session_t *
1144 snat_in2out_lb (snat_main_t *sm,
1151 vlib_node_runtime_t * node)
1153 nat_ed_ses_key_t key;
1154 clib_bihash_kv_16_8_t s_kv, s_value;
1155 udp_header_t *udp = ip4_next_header (ip);
1156 tcp_header_t *tcp = (tcp_header_t *) udp;
1157 snat_session_t *s = 0;
1158 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1159 u32 old_addr, new_addr;
1160 u16 new_port, old_port;
1162 u32 proto = ip_proto_to_snat_proto (ip->protocol);
1163 snat_session_key_t e_key, l_key;
1166 old_addr = ip->src_address.as_u32;
1168 key.l_addr = ip->src_address;
1169 key.r_addr = ip->dst_address;
1170 key.fib_index = rx_fib_index;
1171 key.proto = ip->protocol;
1172 key.r_port = udp->dst_port;
1173 key.l_port = udp->src_port;
1174 s_kv.key[0] = key.as_u64[0];
1175 s_kv.key[1] = key.as_u64[1];
1177 if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1179 s = pool_elt_at_index (tsm->sessions, s_value.value);
1183 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1185 b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1189 l_key.addr = ip->src_address;
1190 l_key.port = udp->src_port;
1191 l_key.protocol = proto;
1192 l_key.fib_index = rx_fib_index;
1193 if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0))
1196 u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1200 clib_warning ("create NAT user failed");
1204 s = nat_session_alloc_or_recycle (sm, u, thread_index);
1207 clib_warning ("create NAT session failed");
1211 s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1212 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1213 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1214 s->outside_address_index = ~0;
1217 u->nstaticsessions++;
1219 /* Add to lookup tables */
1220 s_kv.value = s - tsm->sessions;
1221 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1222 clib_warning ("in2out-ed key add failed");
1224 key.l_addr = e_key.addr;
1225 key.fib_index = e_key.fib_index;
1226 key.l_port = e_key.port;
1227 s_kv.key[0] = key.as_u64[0];
1228 s_kv.key[1] = key.as_u64[1];
1229 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1230 clib_warning ("out2in-ed key add failed");
1233 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1235 /* Update IP checksum */
1237 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1238 if (is_twice_nat_session (s))
1239 sum = ip_csum_update (sum, ip->dst_address.as_u32,
1240 s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1241 ip->checksum = ip_csum_fold (sum);
1243 if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1245 old_port = tcp->src_port;
1246 tcp->src_port = s->out2in.port;
1247 new_port = tcp->src_port;
1249 sum = tcp->checksum;
1250 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1251 sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1252 if (is_twice_nat_session (s))
1254 sum = ip_csum_update (sum, ip->dst_address.as_u32,
1255 s->ext_host_addr.as_u32, ip4_header_t,
1257 sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1258 ip4_header_t, length);
1259 tcp->dst_port = s->ext_host_port;
1260 ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1262 tcp->checksum = ip_csum_fold(sum);
1266 udp->src_port = s->out2in.port;
1267 if (is_twice_nat_session (s))
1269 udp->dst_port = s->ext_host_port;
1270 ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1275 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1276 vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1279 s->last_heard = now;
1281 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1282 /* Per-user LRU list maintenance */
1283 clib_dlist_remove (tsm->list_pool, s->per_user_index);
1284 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1290 snat_in2out_node_fn_inline (vlib_main_t * vm,
1291 vlib_node_runtime_t * node,
1292 vlib_frame_t * frame, int is_slow_path,
1293 int is_output_feature)
1295 u32 n_left_from, * from, * to_next;
1296 snat_in2out_next_t next_index;
1297 u32 pkts_processed = 0;
1298 snat_main_t * sm = &snat_main;
1299 f64 now = vlib_time_now (vm);
1300 u32 stats_node_index;
1301 u32 thread_index = vlib_get_thread_index ();
1303 stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1304 snat_in2out_node.index;
1306 from = vlib_frame_vector_args (frame);
1307 n_left_from = frame->n_vectors;
1308 next_index = node->cached_next_index;
1310 while (n_left_from > 0)
1314 vlib_get_next_frame (vm, node, next_index,
1315 to_next, n_left_to_next);
1317 while (n_left_from >= 4 && n_left_to_next >= 2)
1320 vlib_buffer_t * b0, * b1;
1322 u32 sw_if_index0, sw_if_index1;
1323 ip4_header_t * ip0, * ip1;
1324 ip_csum_t sum0, sum1;
1325 u32 new_addr0, old_addr0, new_addr1, old_addr1;
1326 u16 old_port0, new_port0, old_port1, new_port1;
1327 udp_header_t * udp0, * udp1;
1328 tcp_header_t * tcp0, * tcp1;
1329 icmp46_header_t * icmp0, * icmp1;
1330 snat_session_key_t key0, key1;
1331 u32 rx_fib_index0, rx_fib_index1;
1333 snat_session_t * s0 = 0, * s1 = 0;
1334 clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1335 u32 iph_offset0 = 0, iph_offset1 = 0;
1337 /* Prefetch next iteration. */
1339 vlib_buffer_t * p2, * p3;
1341 p2 = vlib_get_buffer (vm, from[2]);
1342 p3 = vlib_get_buffer (vm, from[3]);
1344 vlib_prefetch_buffer_header (p2, LOAD);
1345 vlib_prefetch_buffer_header (p3, LOAD);
1347 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1348 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1351 /* speculatively enqueue b0 and b1 to the current next frame */
1352 to_next[0] = bi0 = from[0];
1353 to_next[1] = bi1 = from[1];
1357 n_left_to_next -= 2;
1359 b0 = vlib_get_buffer (vm, bi0);
1360 b1 = vlib_get_buffer (vm, bi1);
1362 if (is_output_feature)
1363 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1365 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1368 udp0 = ip4_next_header (ip0);
1369 tcp0 = (tcp_header_t *) udp0;
1370 icmp0 = (icmp46_header_t *) udp0;
1372 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1373 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1376 next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1378 if (PREDICT_FALSE(ip0->ttl == 1))
1380 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1381 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1382 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1384 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1388 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1390 /* Next configured feature, probably ip4-lookup */
1393 if (PREDICT_FALSE (proto0 == ~0))
1395 s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1396 thread_index, now, vm, node);
1398 next0 = SNAT_IN2OUT_NEXT_DROP;
1402 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1404 next0 = icmp_in2out_slow_path
1405 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1406 node, next0, now, thread_index, &s0);
1412 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1414 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1418 if (ip4_is_fragment (ip0))
1420 next0 = SNAT_IN2OUT_NEXT_REASS;
1425 key0.addr = ip0->src_address;
1426 key0.port = udp0->src_port;
1427 key0.protocol = proto0;
1428 key0.fib_index = rx_fib_index0;
1430 kv0.key = key0.as_u64;
1432 if (PREDICT_FALSE (clib_bihash_search_8_8 (
1433 &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1437 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1438 ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
1441 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1442 &s0, node, next0, thread_index);
1443 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1448 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1454 if (PREDICT_FALSE (value0.value == ~0ULL))
1458 s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1459 thread_index, now, vm, node);
1461 next0 = SNAT_IN2OUT_NEXT_DROP;
1466 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1472 s0 = pool_elt_at_index (
1473 sm->per_thread_data[thread_index].sessions,
1478 b0->flags |= VNET_BUFFER_F_IS_NATED;
1480 old_addr0 = ip0->src_address.as_u32;
1481 ip0->src_address = s0->out2in.addr;
1482 new_addr0 = ip0->src_address.as_u32;
1483 if (!is_output_feature)
1484 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1486 sum0 = ip0->checksum;
1487 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1489 src_address /* changed member */);
1490 ip0->checksum = ip_csum_fold (sum0);
1492 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1494 old_port0 = tcp0->src_port;
1495 tcp0->src_port = s0->out2in.port;
1496 new_port0 = tcp0->src_port;
1498 sum0 = tcp0->checksum;
1499 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1501 dst_address /* changed member */);
1502 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1503 ip4_header_t /* cheat */,
1504 length /* changed member */);
1505 tcp0->checksum = ip_csum_fold(sum0);
1509 old_port0 = udp0->src_port;
1510 udp0->src_port = s0->out2in.port;
1515 s0->last_heard = now;
1517 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1518 /* Per-user LRU list maintenance */
1519 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1520 s0->per_user_index);
1521 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1522 s0->per_user_list_head_index,
1523 s0->per_user_index);
1526 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1527 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1529 snat_in2out_trace_t *t =
1530 vlib_add_trace (vm, node, b0, sizeof (*t));
1531 t->is_slow_path = is_slow_path;
1532 t->sw_if_index = sw_if_index0;
1533 t->next_index = next0;
1534 t->session_index = ~0;
1536 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1539 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1541 if (is_output_feature)
1542 iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1544 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1547 udp1 = ip4_next_header (ip1);
1548 tcp1 = (tcp_header_t *) udp1;
1549 icmp1 = (icmp46_header_t *) udp1;
1551 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1552 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1555 if (PREDICT_FALSE(ip1->ttl == 1))
1557 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1558 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1559 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1561 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1565 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1567 /* Next configured feature, probably ip4-lookup */
1570 if (PREDICT_FALSE (proto1 == ~0))
1572 s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1573 thread_index, now, vm, node);
1575 next1 = SNAT_IN2OUT_NEXT_DROP;
1579 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1581 next1 = icmp_in2out_slow_path
1582 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1583 next1, now, thread_index, &s1);
1589 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1591 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1595 if (ip4_is_fragment (ip1))
1597 next1 = SNAT_IN2OUT_NEXT_REASS;
1602 b1->flags |= VNET_BUFFER_F_IS_NATED;
1604 key1.addr = ip1->src_address;
1605 key1.port = udp1->src_port;
1606 key1.protocol = proto1;
1607 key1.fib_index = rx_fib_index1;
1609 kv1.key = key1.as_u64;
1611 if (PREDICT_FALSE(clib_bihash_search_8_8 (
1612 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1616 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1617 ip1, proto1, rx_fib_index1, thread_index)) && !is_output_feature)
1620 next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1621 &s1, node, next1, thread_index);
1622 if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1627 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1633 if (PREDICT_FALSE (value1.value == ~0ULL))
1637 s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1638 thread_index, now, vm, node);
1640 next1 = SNAT_IN2OUT_NEXT_DROP;
1645 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1651 s1 = pool_elt_at_index (
1652 sm->per_thread_data[thread_index].sessions,
1657 old_addr1 = ip1->src_address.as_u32;
1658 ip1->src_address = s1->out2in.addr;
1659 new_addr1 = ip1->src_address.as_u32;
1660 if (!is_output_feature)
1661 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1663 sum1 = ip1->checksum;
1664 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1666 src_address /* changed member */);
1667 ip1->checksum = ip_csum_fold (sum1);
1669 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1671 old_port1 = tcp1->src_port;
1672 tcp1->src_port = s1->out2in.port;
1673 new_port1 = tcp1->src_port;
1675 sum1 = tcp1->checksum;
1676 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1678 dst_address /* changed member */);
1679 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1680 ip4_header_t /* cheat */,
1681 length /* changed member */);
1682 tcp1->checksum = ip_csum_fold(sum1);
1686 old_port1 = udp1->src_port;
1687 udp1->src_port = s1->out2in.port;
1692 s1->last_heard = now;
1694 s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1695 /* Per-user LRU list maintenance */
1696 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1697 s1->per_user_index);
1698 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1699 s1->per_user_list_head_index,
1700 s1->per_user_index);
1703 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1704 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1706 snat_in2out_trace_t *t =
1707 vlib_add_trace (vm, node, b1, sizeof (*t));
1708 t->sw_if_index = sw_if_index1;
1709 t->next_index = next1;
1710 t->session_index = ~0;
1712 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1715 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1717 /* verify speculative enqueues, maybe switch current next frame */
1718 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1719 to_next, n_left_to_next,
1720 bi0, bi1, next0, next1);
1723 while (n_left_from > 0 && n_left_to_next > 0)
1731 u32 new_addr0, old_addr0;
1732 u16 old_port0, new_port0;
1733 udp_header_t * udp0;
1734 tcp_header_t * tcp0;
1735 icmp46_header_t * icmp0;
1736 snat_session_key_t key0;
1739 snat_session_t * s0 = 0;
1740 clib_bihash_kv_8_8_t kv0, value0;
1741 u32 iph_offset0 = 0;
1743 /* speculatively enqueue b0 to the current next frame */
1749 n_left_to_next -= 1;
1751 b0 = vlib_get_buffer (vm, bi0);
1752 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1754 if (is_output_feature)
1755 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1757 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1760 udp0 = ip4_next_header (ip0);
1761 tcp0 = (tcp_header_t *) udp0;
1762 icmp0 = (icmp46_header_t *) udp0;
1764 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1765 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1768 if (PREDICT_FALSE(ip0->ttl == 1))
1770 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1771 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1772 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1774 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1778 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1780 /* Next configured feature, probably ip4-lookup */
1783 if (PREDICT_FALSE (proto0 == ~0))
1785 s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1786 thread_index, now, vm, node);
1788 next0 = SNAT_IN2OUT_NEXT_DROP;
1792 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1794 next0 = icmp_in2out_slow_path
1795 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1796 next0, now, thread_index, &s0);
1802 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1804 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1808 if (ip4_is_fragment (ip0))
1810 next0 = SNAT_IN2OUT_NEXT_REASS;
1815 key0.addr = ip0->src_address;
1816 key0.port = udp0->src_port;
1817 key0.protocol = proto0;
1818 key0.fib_index = rx_fib_index0;
1820 kv0.key = key0.as_u64;
1822 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1827 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1828 ip0, proto0, rx_fib_index0, thread_index)) && !is_output_feature)
1831 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1832 &s0, node, next0, thread_index);
1834 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1839 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1845 if (PREDICT_FALSE (value0.value == ~0ULL))
1849 s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1850 thread_index, now, vm, node);
1852 next0 = SNAT_IN2OUT_NEXT_DROP;
1857 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1863 s0 = pool_elt_at_index (
1864 sm->per_thread_data[thread_index].sessions,
1869 b0->flags |= VNET_BUFFER_F_IS_NATED;
1871 old_addr0 = ip0->src_address.as_u32;
1872 ip0->src_address = s0->out2in.addr;
1873 new_addr0 = ip0->src_address.as_u32;
1874 if (!is_output_feature)
1875 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1877 sum0 = ip0->checksum;
1878 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1880 src_address /* changed member */);
1881 ip0->checksum = ip_csum_fold (sum0);
1883 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1885 old_port0 = tcp0->src_port;
1886 tcp0->src_port = s0->out2in.port;
1887 new_port0 = tcp0->src_port;
1889 sum0 = tcp0->checksum;
1890 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1892 dst_address /* changed member */);
1893 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1894 ip4_header_t /* cheat */,
1895 length /* changed member */);
1896 tcp0->checksum = ip_csum_fold(sum0);
1900 old_port0 = udp0->src_port;
1901 udp0->src_port = s0->out2in.port;
1906 s0->last_heard = now;
1908 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1909 /* Per-user LRU list maintenance */
1910 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1911 s0->per_user_index);
1912 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1913 s0->per_user_list_head_index,
1914 s0->per_user_index);
1917 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1918 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1920 snat_in2out_trace_t *t =
1921 vlib_add_trace (vm, node, b0, sizeof (*t));
1922 t->is_slow_path = is_slow_path;
1923 t->sw_if_index = sw_if_index0;
1924 t->next_index = next0;
1925 t->session_index = ~0;
1927 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1930 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1932 /* verify speculative enqueue, maybe switch current next frame */
1933 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1934 to_next, n_left_to_next,
1938 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1941 vlib_node_increment_counter (vm, stats_node_index,
1942 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1944 return frame->n_vectors;
1948 snat_in2out_fast_path_fn (vlib_main_t * vm,
1949 vlib_node_runtime_t * node,
1950 vlib_frame_t * frame)
1952 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1955 VLIB_REGISTER_NODE (snat_in2out_node) = {
1956 .function = snat_in2out_fast_path_fn,
1957 .name = "nat44-in2out",
1958 .vector_size = sizeof (u32),
1959 .format_trace = format_snat_in2out_trace,
1960 .type = VLIB_NODE_TYPE_INTERNAL,
1962 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1963 .error_strings = snat_in2out_error_strings,
1965 .runtime_data_bytes = sizeof (snat_runtime_t),
1967 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1969 /* edit / add dispositions here */
1971 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1972 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1973 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1974 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1975 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1979 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1982 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1983 vlib_node_runtime_t * node,
1984 vlib_frame_t * frame)
1986 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1989 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1990 .function = snat_in2out_output_fast_path_fn,
1991 .name = "nat44-in2out-output",
1992 .vector_size = sizeof (u32),
1993 .format_trace = format_snat_in2out_trace,
1994 .type = VLIB_NODE_TYPE_INTERNAL,
1996 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1997 .error_strings = snat_in2out_error_strings,
1999 .runtime_data_bytes = sizeof (snat_runtime_t),
2001 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2003 /* edit / add dispositions here */
2005 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2006 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2007 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2008 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2009 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2013 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2014 snat_in2out_output_fast_path_fn);
2017 snat_in2out_slow_path_fn (vlib_main_t * vm,
2018 vlib_node_runtime_t * node,
2019 vlib_frame_t * frame)
2021 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2024 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2025 .function = snat_in2out_slow_path_fn,
2026 .name = "nat44-in2out-slowpath",
2027 .vector_size = sizeof (u32),
2028 .format_trace = format_snat_in2out_trace,
2029 .type = VLIB_NODE_TYPE_INTERNAL,
2031 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2032 .error_strings = snat_in2out_error_strings,
2034 .runtime_data_bytes = sizeof (snat_runtime_t),
2036 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2038 /* edit / add dispositions here */
2040 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2041 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2042 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2043 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2044 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2048 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2049 snat_in2out_slow_path_fn);
2052 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2053 vlib_node_runtime_t * node,
2054 vlib_frame_t * frame)
2056 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2059 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2060 .function = snat_in2out_output_slow_path_fn,
2061 .name = "nat44-in2out-output-slowpath",
2062 .vector_size = sizeof (u32),
2063 .format_trace = format_snat_in2out_trace,
2064 .type = VLIB_NODE_TYPE_INTERNAL,
2066 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2067 .error_strings = snat_in2out_error_strings,
2069 .runtime_data_bytes = sizeof (snat_runtime_t),
2071 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2073 /* edit / add dispositions here */
2075 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2076 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2077 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2078 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2079 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2083 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2084 snat_in2out_output_slow_path_fn);
2086 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2089 nat44_hairpinning_fn (vlib_main_t * vm,
2090 vlib_node_runtime_t * node,
2091 vlib_frame_t * frame)
2093 u32 n_left_from, * from, * to_next;
2094 snat_in2out_next_t next_index;
2095 u32 pkts_processed = 0;
2096 snat_main_t * sm = &snat_main;
2097 vnet_feature_main_t *fm = &feature_main;
2098 u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2099 vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2101 from = vlib_frame_vector_args (frame);
2102 n_left_from = frame->n_vectors;
2103 next_index = node->cached_next_index;
2105 while (n_left_from > 0)
2109 vlib_get_next_frame (vm, node, next_index,
2110 to_next, n_left_to_next);
2112 while (n_left_from > 0 && n_left_to_next > 0)
2119 udp_header_t * udp0;
2120 tcp_header_t * tcp0;
2122 /* speculatively enqueue b0 to the current next frame */
2128 n_left_to_next -= 1;
2130 b0 = vlib_get_buffer (vm, bi0);
2131 ip0 = vlib_buffer_get_current (b0);
2132 udp0 = ip4_next_header (ip0);
2133 tcp0 = (tcp_header_t *) udp0;
2135 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2137 vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2140 if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2141 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2143 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2145 /* verify speculative enqueue, maybe switch current next frame */
2146 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2147 to_next, n_left_to_next,
2151 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2154 vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2155 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2157 return frame->n_vectors;
2160 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2161 .function = nat44_hairpinning_fn,
2162 .name = "nat44-hairpinning",
2163 .vector_size = sizeof (u32),
2164 .type = VLIB_NODE_TYPE_INTERNAL,
2165 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2166 .error_strings = snat_in2out_error_strings,
2169 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2170 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2174 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2175 nat44_hairpinning_fn);
2178 nat44_reass_hairpinning (snat_main_t *sm,
2185 snat_session_key_t key0, sm0;
2186 snat_session_t * s0;
2187 clib_bihash_kv_8_8_t kv0, value0;
2189 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2190 u16 new_dst_port0, old_dst_port0;
2191 udp_header_t * udp0;
2192 tcp_header_t * tcp0;
2194 key0.addr = ip0->dst_address;
2196 key0.protocol = proto0;
2197 key0.fib_index = sm->outside_fib_index;
2198 kv0.key = key0.as_u64;
2200 udp0 = ip4_next_header (ip0);
2202 /* Check if destination is static mappings */
2203 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
2205 new_dst_addr0 = sm0.addr.as_u32;
2206 new_dst_port0 = sm0.port;
2207 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2209 /* or active sessions */
2212 if (sm->num_workers > 1)
2213 ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2215 ti = sm->num_workers;
2217 if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2220 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2221 new_dst_addr0 = s0->in2out.addr.as_u32;
2222 new_dst_port0 = s0->in2out.port;
2223 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2227 /* Destination is behind the same NAT, use internal address and port */
2230 old_dst_addr0 = ip0->dst_address.as_u32;
2231 ip0->dst_address.as_u32 = new_dst_addr0;
2232 sum0 = ip0->checksum;
2233 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2234 ip4_header_t, dst_address);
2235 ip0->checksum = ip_csum_fold (sum0);
2237 old_dst_port0 = dport;
2238 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2239 ip4_is_first_fragment (ip0)))
2241 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2243 tcp0 = ip4_next_header (ip0);
2244 tcp0->dst = new_dst_port0;
2245 sum0 = tcp0->checksum;
2246 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2247 ip4_header_t, dst_address);
2248 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2249 ip4_header_t /* cheat */, length);
2250 tcp0->checksum = ip_csum_fold(sum0);
2254 udp0->dst_port = new_dst_port0;
2260 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2262 tcp0 = ip4_next_header (ip0);
2263 sum0 = tcp0->checksum;
2264 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2265 ip4_header_t, dst_address);
2266 tcp0->checksum = ip_csum_fold(sum0);
2273 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2274 vlib_node_runtime_t * node,
2275 vlib_frame_t * frame)
2277 u32 n_left_from, *from, *to_next;
2278 snat_in2out_next_t next_index;
2279 u32 pkts_processed = 0;
2280 snat_main_t *sm = &snat_main;
2281 f64 now = vlib_time_now (vm);
2282 u32 thread_index = vlib_get_thread_index ();
2283 snat_main_per_thread_data_t *per_thread_data =
2284 &sm->per_thread_data[thread_index];
2285 u32 *fragments_to_drop = 0;
2286 u32 *fragments_to_loopback = 0;
2288 from = vlib_frame_vector_args (frame);
2289 n_left_from = frame->n_vectors;
2290 next_index = node->cached_next_index;
2292 while (n_left_from > 0)
2296 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2298 while (n_left_from > 0 && n_left_to_next > 0)
2300 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2305 nat_reass_ip4_t *reass0;
2306 udp_header_t * udp0;
2307 tcp_header_t * tcp0;
2308 snat_session_key_t key0;
2309 clib_bihash_kv_8_8_t kv0, value0;
2310 snat_session_t * s0 = 0;
2311 u16 old_port0, new_port0;
2314 /* speculatively enqueue b0 to the current next frame */
2320 n_left_to_next -= 1;
2322 b0 = vlib_get_buffer (vm, bi0);
2323 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2325 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2326 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2329 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2331 next0 = SNAT_IN2OUT_NEXT_DROP;
2332 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2336 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2337 udp0 = ip4_next_header (ip0);
2338 tcp0 = (tcp_header_t *) udp0;
2339 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2341 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2346 &fragments_to_drop);
2348 if (PREDICT_FALSE (!reass0))
2350 next0 = SNAT_IN2OUT_NEXT_DROP;
2351 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2355 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2357 key0.addr = ip0->src_address;
2358 key0.port = udp0->src_port;
2359 key0.protocol = proto0;
2360 key0.fib_index = rx_fib_index0;
2361 kv0.key = key0.as_u64;
2363 if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2365 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2366 ip0, proto0, rx_fib_index0, thread_index)))
2369 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2370 &s0, node, next0, thread_index);
2372 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2375 reass0->sess_index = s0 - per_thread_data->sessions;
2379 s0 = pool_elt_at_index (per_thread_data->sessions,
2381 reass0->sess_index = value0.value;
2383 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2387 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2389 if (nat_ip4_reass_add_fragment (reass0, bi0))
2391 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2392 next0 = SNAT_IN2OUT_NEXT_DROP;
2398 s0 = pool_elt_at_index (per_thread_data->sessions,
2399 reass0->sess_index);
2402 old_addr0 = ip0->src_address.as_u32;
2403 ip0->src_address = s0->out2in.addr;
2404 new_addr0 = ip0->src_address.as_u32;
2405 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2407 sum0 = ip0->checksum;
2408 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2410 src_address /* changed member */);
2411 ip0->checksum = ip_csum_fold (sum0);
2413 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2415 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2417 old_port0 = tcp0->src_port;
2418 tcp0->src_port = s0->out2in.port;
2419 new_port0 = tcp0->src_port;
2421 sum0 = tcp0->checksum;
2422 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2424 dst_address /* changed member */);
2425 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2426 ip4_header_t /* cheat */,
2427 length /* changed member */);
2428 tcp0->checksum = ip_csum_fold(sum0);
2432 old_port0 = udp0->src_port;
2433 udp0->src_port = s0->out2in.port;
2439 nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2440 s0->ext_host_port, proto0);
2443 s0->last_heard = now;
2445 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2446 /* Per-user LRU list maintenance */
2447 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2448 s0->per_user_index);
2449 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2450 s0->per_user_list_head_index,
2451 s0->per_user_index);
2454 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2455 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2457 nat44_in2out_reass_trace_t *t =
2458 vlib_add_trace (vm, node, b0, sizeof (*t));
2459 t->cached = cached0;
2460 t->sw_if_index = sw_if_index0;
2461 t->next_index = next0;
2471 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2473 /* verify speculative enqueue, maybe switch current next frame */
2474 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2475 to_next, n_left_to_next,
2479 if (n_left_from == 0 && vec_len (fragments_to_loopback))
2481 from = vlib_frame_vector_args (frame);
2482 u32 len = vec_len (fragments_to_loopback);
2483 if (len <= VLIB_FRAME_SIZE)
2485 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2487 vec_reset_length (fragments_to_loopback);
2492 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2493 sizeof (u32) * VLIB_FRAME_SIZE);
2494 n_left_from = VLIB_FRAME_SIZE;
2495 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2500 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2503 vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2504 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2507 nat_send_all_to_node (vm, fragments_to_drop, node,
2508 &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2509 SNAT_IN2OUT_NEXT_DROP);
2511 vec_free (fragments_to_drop);
2512 vec_free (fragments_to_loopback);
2513 return frame->n_vectors;
2516 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2517 .function = nat44_in2out_reass_node_fn,
2518 .name = "nat44-in2out-reass",
2519 .vector_size = sizeof (u32),
2520 .format_trace = format_nat44_in2out_reass_trace,
2521 .type = VLIB_NODE_TYPE_INTERNAL,
2523 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2524 .error_strings = snat_in2out_error_strings,
2526 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2528 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2529 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2530 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2531 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2532 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2536 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2537 nat44_in2out_reass_node_fn);
2539 /**************************/
2540 /*** deterministic mode ***/
2541 /**************************/
2543 snat_det_in2out_node_fn (vlib_main_t * vm,
2544 vlib_node_runtime_t * node,
2545 vlib_frame_t * frame)
2547 u32 n_left_from, * from, * to_next;
2548 snat_in2out_next_t next_index;
2549 u32 pkts_processed = 0;
2550 snat_main_t * sm = &snat_main;
2551 u32 now = (u32) vlib_time_now (vm);
2552 u32 thread_index = vlib_get_thread_index ();
2554 from = vlib_frame_vector_args (frame);
2555 n_left_from = frame->n_vectors;
2556 next_index = node->cached_next_index;
2558 while (n_left_from > 0)
2562 vlib_get_next_frame (vm, node, next_index,
2563 to_next, n_left_to_next);
2565 while (n_left_from >= 4 && n_left_to_next >= 2)
2568 vlib_buffer_t * b0, * b1;
2570 u32 sw_if_index0, sw_if_index1;
2571 ip4_header_t * ip0, * ip1;
2572 ip_csum_t sum0, sum1;
2573 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2574 u16 old_port0, new_port0, lo_port0, i0;
2575 u16 old_port1, new_port1, lo_port1, i1;
2576 udp_header_t * udp0, * udp1;
2577 tcp_header_t * tcp0, * tcp1;
2579 snat_det_out_key_t key0, key1;
2580 snat_det_map_t * dm0, * dm1;
2581 snat_det_session_t * ses0 = 0, * ses1 = 0;
2582 u32 rx_fib_index0, rx_fib_index1;
2583 icmp46_header_t * icmp0, * icmp1;
2585 /* Prefetch next iteration. */
2587 vlib_buffer_t * p2, * p3;
2589 p2 = vlib_get_buffer (vm, from[2]);
2590 p3 = vlib_get_buffer (vm, from[3]);
2592 vlib_prefetch_buffer_header (p2, LOAD);
2593 vlib_prefetch_buffer_header (p3, LOAD);
2595 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2596 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2599 /* speculatively enqueue b0 and b1 to the current next frame */
2600 to_next[0] = bi0 = from[0];
2601 to_next[1] = bi1 = from[1];
2605 n_left_to_next -= 2;
2607 b0 = vlib_get_buffer (vm, bi0);
2608 b1 = vlib_get_buffer (vm, bi1);
2610 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2611 next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2613 ip0 = vlib_buffer_get_current (b0);
2614 udp0 = ip4_next_header (ip0);
2615 tcp0 = (tcp_header_t *) udp0;
2617 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2619 if (PREDICT_FALSE(ip0->ttl == 1))
2621 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2622 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2623 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2625 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2629 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2631 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2633 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2634 icmp0 = (icmp46_header_t *) udp0;
2636 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2637 rx_fib_index0, node, next0, thread_index,
2642 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2643 if (PREDICT_FALSE(!dm0))
2645 clib_warning("no match for internal host %U",
2646 format_ip4_address, &ip0->src_address);
2647 next0 = SNAT_IN2OUT_NEXT_DROP;
2648 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2652 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2654 key0.ext_host_addr = ip0->dst_address;
2655 key0.ext_host_port = tcp0->dst;
2657 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2658 if (PREDICT_FALSE(!ses0))
2660 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2662 key0.out_port = clib_host_to_net_u16 (lo_port0 +
2663 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2665 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2668 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2671 if (PREDICT_FALSE(!ses0))
2673 /* too many sessions for user, send ICMP error packet */
2675 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2676 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2677 ICMP4_destination_unreachable_destination_unreachable_host,
2679 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2684 new_port0 = ses0->out.out_port;
2686 old_addr0.as_u32 = ip0->src_address.as_u32;
2687 ip0->src_address.as_u32 = new_addr0.as_u32;
2688 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2690 sum0 = ip0->checksum;
2691 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2693 src_address /* changed member */);
2694 ip0->checksum = ip_csum_fold (sum0);
2696 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2698 if (tcp0->flags & TCP_FLAG_SYN)
2699 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2700 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2701 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2702 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2703 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2704 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2705 snat_det_ses_close(dm0, ses0);
2706 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2707 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2708 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2709 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2711 old_port0 = tcp0->src;
2712 tcp0->src = new_port0;
2714 sum0 = tcp0->checksum;
2715 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2717 dst_address /* changed member */);
2718 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2719 ip4_header_t /* cheat */,
2720 length /* changed member */);
2721 tcp0->checksum = ip_csum_fold(sum0);
2725 ses0->state = SNAT_SESSION_UDP_ACTIVE;
2726 old_port0 = udp0->src_port;
2727 udp0->src_port = new_port0;
2733 case SNAT_SESSION_UDP_ACTIVE:
2734 ses0->expire = now + sm->udp_timeout;
2736 case SNAT_SESSION_TCP_SYN_SENT:
2737 case SNAT_SESSION_TCP_FIN_WAIT:
2738 case SNAT_SESSION_TCP_CLOSE_WAIT:
2739 case SNAT_SESSION_TCP_LAST_ACK:
2740 ses0->expire = now + sm->tcp_transitory_timeout;
2742 case SNAT_SESSION_TCP_ESTABLISHED:
2743 ses0->expire = now + sm->tcp_established_timeout;
2748 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2749 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2751 snat_in2out_trace_t *t =
2752 vlib_add_trace (vm, node, b0, sizeof (*t));
2753 t->is_slow_path = 0;
2754 t->sw_if_index = sw_if_index0;
2755 t->next_index = next0;
2756 t->session_index = ~0;
2758 t->session_index = ses0 - dm0->sessions;
2761 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2763 ip1 = vlib_buffer_get_current (b1);
2764 udp1 = ip4_next_header (ip1);
2765 tcp1 = (tcp_header_t *) udp1;
2767 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2769 if (PREDICT_FALSE(ip1->ttl == 1))
2771 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2772 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2773 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2775 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2779 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2781 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2783 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2784 icmp1 = (icmp46_header_t *) udp1;
2786 next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2787 rx_fib_index1, node, next1, thread_index,
2792 dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2793 if (PREDICT_FALSE(!dm1))
2795 clib_warning("no match for internal host %U",
2796 format_ip4_address, &ip0->src_address);
2797 next1 = SNAT_IN2OUT_NEXT_DROP;
2798 b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2802 snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
2804 key1.ext_host_addr = ip1->dst_address;
2805 key1.ext_host_port = tcp1->dst;
2807 ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
2808 if (PREDICT_FALSE(!ses1))
2810 for (i1 = 0; i1 < dm1->ports_per_host; i1++)
2812 key1.out_port = clib_host_to_net_u16 (lo_port1 +
2813 ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
2815 if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
2818 ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
2821 if (PREDICT_FALSE(!ses1))
2823 /* too many sessions for user, send ICMP error packet */
2825 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2826 icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
2827 ICMP4_destination_unreachable_destination_unreachable_host,
2829 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2834 new_port1 = ses1->out.out_port;
2836 old_addr1.as_u32 = ip1->src_address.as_u32;
2837 ip1->src_address.as_u32 = new_addr1.as_u32;
2838 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2840 sum1 = ip1->checksum;
2841 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2843 src_address /* changed member */);
2844 ip1->checksum = ip_csum_fold (sum1);
2846 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2848 if (tcp1->flags & TCP_FLAG_SYN)
2849 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
2850 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
2851 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2852 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2853 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
2854 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
2855 snat_det_ses_close(dm1, ses1);
2856 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2857 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
2858 else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
2859 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
2861 old_port1 = tcp1->src;
2862 tcp1->src = new_port1;
2864 sum1 = tcp1->checksum;
2865 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2867 dst_address /* changed member */);
2868 sum1 = ip_csum_update (sum1, old_port1, new_port1,
2869 ip4_header_t /* cheat */,
2870 length /* changed member */);
2871 tcp1->checksum = ip_csum_fold(sum1);
2875 ses1->state = SNAT_SESSION_UDP_ACTIVE;
2876 old_port1 = udp1->src_port;
2877 udp1->src_port = new_port1;
2883 case SNAT_SESSION_UDP_ACTIVE:
2884 ses1->expire = now + sm->udp_timeout;
2886 case SNAT_SESSION_TCP_SYN_SENT:
2887 case SNAT_SESSION_TCP_FIN_WAIT:
2888 case SNAT_SESSION_TCP_CLOSE_WAIT:
2889 case SNAT_SESSION_TCP_LAST_ACK:
2890 ses1->expire = now + sm->tcp_transitory_timeout;
2892 case SNAT_SESSION_TCP_ESTABLISHED:
2893 ses1->expire = now + sm->tcp_established_timeout;
2898 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2899 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2901 snat_in2out_trace_t *t =
2902 vlib_add_trace (vm, node, b1, sizeof (*t));
2903 t->is_slow_path = 0;
2904 t->sw_if_index = sw_if_index1;
2905 t->next_index = next1;
2906 t->session_index = ~0;
2908 t->session_index = ses1 - dm1->sessions;
2911 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
2913 /* verify speculative enqueues, maybe switch current next frame */
2914 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2915 to_next, n_left_to_next,
2916 bi0, bi1, next0, next1);
2919 while (n_left_from > 0 && n_left_to_next > 0)
2927 ip4_address_t new_addr0, old_addr0;
2928 u16 old_port0, new_port0, lo_port0, i0;
2929 udp_header_t * udp0;
2930 tcp_header_t * tcp0;
2932 snat_det_out_key_t key0;
2933 snat_det_map_t * dm0;
2934 snat_det_session_t * ses0 = 0;
2936 icmp46_header_t * icmp0;
2938 /* speculatively enqueue b0 to the current next frame */
2944 n_left_to_next -= 1;
2946 b0 = vlib_get_buffer (vm, bi0);
2947 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2949 ip0 = vlib_buffer_get_current (b0);
2950 udp0 = ip4_next_header (ip0);
2951 tcp0 = (tcp_header_t *) udp0;
2953 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2955 if (PREDICT_FALSE(ip0->ttl == 1))
2957 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2958 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2959 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2961 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2965 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2967 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2969 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2970 icmp0 = (icmp46_header_t *) udp0;
2972 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2973 rx_fib_index0, node, next0, thread_index,
2978 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2979 if (PREDICT_FALSE(!dm0))
2981 clib_warning("no match for internal host %U",
2982 format_ip4_address, &ip0->src_address);
2983 next0 = SNAT_IN2OUT_NEXT_DROP;
2984 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2988 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2990 key0.ext_host_addr = ip0->dst_address;
2991 key0.ext_host_port = tcp0->dst;
2993 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2994 if (PREDICT_FALSE(!ses0))
2996 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2998 key0.out_port = clib_host_to_net_u16 (lo_port0 +
2999 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3001 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3004 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3007 if (PREDICT_FALSE(!ses0))
3009 /* too many sessions for user, send ICMP error packet */
3011 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3012 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3013 ICMP4_destination_unreachable_destination_unreachable_host,
3015 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3020 new_port0 = ses0->out.out_port;
3022 old_addr0.as_u32 = ip0->src_address.as_u32;
3023 ip0->src_address.as_u32 = new_addr0.as_u32;
3024 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3026 sum0 = ip0->checksum;
3027 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3029 src_address /* changed member */);
3030 ip0->checksum = ip_csum_fold (sum0);
3032 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3034 if (tcp0->flags & TCP_FLAG_SYN)
3035 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3036 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3037 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3038 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3039 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3040 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3041 snat_det_ses_close(dm0, ses0);
3042 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3043 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3044 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3045 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3047 old_port0 = tcp0->src;
3048 tcp0->src = new_port0;
3050 sum0 = tcp0->checksum;
3051 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3053 dst_address /* changed member */);
3054 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3055 ip4_header_t /* cheat */,
3056 length /* changed member */);
3057 tcp0->checksum = ip_csum_fold(sum0);
3061 ses0->state = SNAT_SESSION_UDP_ACTIVE;
3062 old_port0 = udp0->src_port;
3063 udp0->src_port = new_port0;
3069 case SNAT_SESSION_UDP_ACTIVE:
3070 ses0->expire = now + sm->udp_timeout;
3072 case SNAT_SESSION_TCP_SYN_SENT:
3073 case SNAT_SESSION_TCP_FIN_WAIT:
3074 case SNAT_SESSION_TCP_CLOSE_WAIT:
3075 case SNAT_SESSION_TCP_LAST_ACK:
3076 ses0->expire = now + sm->tcp_transitory_timeout;
3078 case SNAT_SESSION_TCP_ESTABLISHED:
3079 ses0->expire = now + sm->tcp_established_timeout;
3084 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3085 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3087 snat_in2out_trace_t *t =
3088 vlib_add_trace (vm, node, b0, sizeof (*t));
3089 t->is_slow_path = 0;
3090 t->sw_if_index = sw_if_index0;
3091 t->next_index = next0;
3092 t->session_index = ~0;
3094 t->session_index = ses0 - dm0->sessions;
3097 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3099 /* verify speculative enqueue, maybe switch current next frame */
3100 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3101 to_next, n_left_to_next,
3105 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3108 vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3109 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3111 return frame->n_vectors;
3114 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3115 .function = snat_det_in2out_node_fn,
3116 .name = "nat44-det-in2out",
3117 .vector_size = sizeof (u32),
3118 .format_trace = format_snat_in2out_trace,
3119 .type = VLIB_NODE_TYPE_INTERNAL,
3121 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3122 .error_strings = snat_in2out_error_strings,
3124 .runtime_data_bytes = sizeof (snat_runtime_t),
3128 /* edit / add dispositions here */
3130 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3131 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3132 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3136 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3139 * Get address and port values to be used for ICMP packet translation
3140 * and create session if needed
3142 * @param[in,out] sm NAT main
3143 * @param[in,out] node NAT node runtime
3144 * @param[in] thread_index thread index
3145 * @param[in,out] b0 buffer containing packet to be translated
3146 * @param[out] p_proto protocol used for matching
3147 * @param[out] p_value address and port after NAT translation
3148 * @param[out] p_dont_translate if packet should not be translated
3149 * @param d optional parameter
3150 * @param e optional parameter
3152 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3153 u32 thread_index, vlib_buffer_t *b0,
3154 ip4_header_t *ip0, u8 *p_proto,
3155 snat_session_key_t *p_value,
3156 u8 *p_dont_translate, void *d, void *e)
3158 icmp46_header_t *icmp0;
3162 snat_det_out_key_t key0;
3163 u8 dont_translate = 0;
3165 icmp_echo_header_t *echo0, *inner_echo0 = 0;
3166 ip4_header_t *inner_ip0;
3167 void *l4_header = 0;
3168 icmp46_header_t *inner_icmp0;
3169 snat_det_map_t * dm0 = 0;
3170 ip4_address_t new_addr0;
3172 snat_det_session_t * ses0 = 0;
3173 ip4_address_t in_addr;
3176 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3177 echo0 = (icmp_echo_header_t *)(icmp0+1);
3178 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3179 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3181 if (!icmp_is_error_message (icmp0))
3183 protocol = SNAT_PROTOCOL_ICMP;
3184 in_addr = ip0->src_address;
3185 in_port = echo0->identifier;
3189 inner_ip0 = (ip4_header_t *)(echo0+1);
3190 l4_header = ip4_next_header (inner_ip0);
3191 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3192 in_addr = inner_ip0->dst_address;
3195 case SNAT_PROTOCOL_ICMP:
3196 inner_icmp0 = (icmp46_header_t*)l4_header;
3197 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3198 in_port = inner_echo0->identifier;
3200 case SNAT_PROTOCOL_UDP:
3201 case SNAT_PROTOCOL_TCP:
3202 in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3205 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3206 next0 = SNAT_IN2OUT_NEXT_DROP;
3211 dm0 = snat_det_map_by_user(sm, &in_addr);
3212 if (PREDICT_FALSE(!dm0))
3214 clib_warning("no match for internal host %U",
3215 format_ip4_address, &in_addr);
3216 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3217 IP_PROTOCOL_ICMP, rx_fib_index0)))
3222 next0 = SNAT_IN2OUT_NEXT_DROP;
3223 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3227 snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3229 key0.ext_host_addr = ip0->dst_address;
3230 key0.ext_host_port = 0;
3232 ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3233 if (PREDICT_FALSE(!ses0))
3235 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3236 IP_PROTOCOL_ICMP, rx_fib_index0)))
3241 if (icmp0->type != ICMP4_echo_request)
3243 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3244 next0 = SNAT_IN2OUT_NEXT_DROP;
3247 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3249 key0.out_port = clib_host_to_net_u16 (lo_port0 +
3250 ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3252 if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3255 ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3258 if (PREDICT_FALSE(!ses0))
3260 next0 = SNAT_IN2OUT_NEXT_DROP;
3261 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3266 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3267 !icmp_is_error_message (icmp0)))
3269 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3270 next0 = SNAT_IN2OUT_NEXT_DROP;
3274 u32 now = (u32) vlib_time_now (sm->vlib_main);
3276 ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3277 ses0->expire = now + sm->icmp_timeout;
3280 *p_proto = protocol;
3283 p_value->addr = new_addr0;
3284 p_value->fib_index = sm->outside_fib_index;
3285 p_value->port = ses0->out.out_port;
3287 *p_dont_translate = dont_translate;
3289 *(snat_det_session_t**)d = ses0;
3291 *(snat_det_map_t**)e = dm0;
3295 /**********************/
3296 /*** worker handoff ***/
3297 /**********************/
3299 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3300 vlib_node_runtime_t * node,
3301 vlib_frame_t * frame,
3304 snat_main_t *sm = &snat_main;
3305 vlib_thread_main_t *tm = vlib_get_thread_main ();
3306 u32 n_left_from, *from, *to_next = 0;
3307 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3308 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3310 vlib_frame_queue_elt_t *hf = 0;
3311 vlib_frame_t *f = 0;
3313 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3314 u32 next_worker_index = 0;
3315 u32 current_worker_index = ~0;
3316 u32 thread_index = vlib_get_thread_index ();
3320 ASSERT (vec_len (sm->workers));
3324 fq_index = sm->fq_in2out_output_index;
3325 to_node_index = sm->in2out_output_node_index;
3329 fq_index = sm->fq_in2out_index;
3330 to_node_index = sm->in2out_node_index;
3333 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3335 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3337 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3338 sm->first_worker_index + sm->num_workers - 1,
3339 (vlib_frame_queue_t *) (~0));
3342 from = vlib_frame_vector_args (frame);
3343 n_left_from = frame->n_vectors;
3345 while (n_left_from > 0)
3358 b0 = vlib_get_buffer (vm, bi0);
3360 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3361 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3363 ip0 = vlib_buffer_get_current (b0);
3365 next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3367 if (PREDICT_FALSE (next_worker_index != thread_index))
3371 if (next_worker_index != current_worker_index)
3374 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3376 hf = vlib_get_worker_handoff_queue_elt (fq_index,
3378 handoff_queue_elt_by_worker_index);
3380 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3381 to_next_worker = &hf->buffer_index[hf->n_vectors];
3382 current_worker_index = next_worker_index;
3385 /* enqueue to correct worker thread */
3386 to_next_worker[0] = bi0;
3388 n_left_to_next_worker--;
3390 if (n_left_to_next_worker == 0)
3392 hf->n_vectors = VLIB_FRAME_SIZE;
3393 vlib_put_frame_queue_elt (hf);
3394 current_worker_index = ~0;
3395 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3402 /* if this is 1st frame */
3405 f = vlib_get_frame_to_node (vm, to_node_index);
3406 to_next = vlib_frame_vector_args (f);
3414 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3415 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3417 snat_in2out_worker_handoff_trace_t *t =
3418 vlib_add_trace (vm, node, b0, sizeof (*t));
3419 t->next_worker_index = next_worker_index;
3420 t->do_handoff = do_handoff;
3425 vlib_put_frame_to_node (vm, to_node_index, f);
3428 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3430 /* Ship frames to the worker nodes */
3431 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3433 if (handoff_queue_elt_by_worker_index[i])
3435 hf = handoff_queue_elt_by_worker_index[i];
3437 * It works better to let the handoff node
3438 * rate-adapt, always ship the handoff queue element.
3440 if (1 || hf->n_vectors == hf->last_n_vectors)
3442 vlib_put_frame_queue_elt (hf);
3443 handoff_queue_elt_by_worker_index[i] = 0;
3446 hf->last_n_vectors = hf->n_vectors;
3448 congested_handoff_queue_by_worker_index[i] =
3449 (vlib_frame_queue_t *) (~0);
3452 current_worker_index = ~0;
3453 return frame->n_vectors;
3457 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3458 vlib_node_runtime_t * node,
3459 vlib_frame_t * frame)
3461 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3464 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3465 .function = snat_in2out_worker_handoff_fn,
3466 .name = "nat44-in2out-worker-handoff",
3467 .vector_size = sizeof (u32),
3468 .format_trace = format_snat_in2out_worker_handoff_trace,
3469 .type = VLIB_NODE_TYPE_INTERNAL,
3478 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3479 snat_in2out_worker_handoff_fn);
3482 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3483 vlib_node_runtime_t * node,
3484 vlib_frame_t * frame)
3486 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3489 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3490 .function = snat_in2out_output_worker_handoff_fn,
3491 .name = "nat44-in2out-output-worker-handoff",
3492 .vector_size = sizeof (u32),
3493 .format_trace = format_snat_in2out_worker_handoff_trace,
3494 .type = VLIB_NODE_TYPE_INTERNAL,
3503 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3504 snat_in2out_output_worker_handoff_fn);
3506 static_always_inline int
3507 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3509 snat_address_t * ap;
3510 clib_bihash_kv_8_8_t kv, value;
3511 snat_session_key_t m_key;
3513 vec_foreach (ap, sm->addresses)
3515 if (ap->addr.as_u32 == dst_addr->as_u32)
3519 m_key.addr.as_u32 = dst_addr->as_u32;
3520 m_key.fib_index = sm->outside_fib_index;
3523 kv.key = m_key.as_u64;
3524 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3531 snat_hairpin_dst_fn (vlib_main_t * vm,
3532 vlib_node_runtime_t * node,
3533 vlib_frame_t * frame)
3535 u32 n_left_from, * from, * to_next;
3536 snat_in2out_next_t next_index;
3537 u32 pkts_processed = 0;
3538 snat_main_t * sm = &snat_main;
3540 from = vlib_frame_vector_args (frame);
3541 n_left_from = frame->n_vectors;
3542 next_index = node->cached_next_index;
3544 while (n_left_from > 0)
3548 vlib_get_next_frame (vm, node, next_index,
3549 to_next, n_left_to_next);
3551 while (n_left_from > 0 && n_left_to_next > 0)
3559 /* speculatively enqueue b0 to the current next frame */
3565 n_left_to_next -= 1;
3567 b0 = vlib_get_buffer (vm, bi0);
3568 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3569 ip0 = vlib_buffer_get_current (b0);
3571 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3573 vnet_buffer (b0)->snat.flags = 0;
3574 if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3576 if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3578 udp_header_t * udp0 = ip4_next_header (ip0);
3579 tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3581 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3583 else if (proto0 == SNAT_PROTOCOL_ICMP)
3585 icmp46_header_t * icmp0 = ip4_next_header (ip0);
3587 snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3591 snat_hairpinning_unknown_proto (sm, b0, ip0);
3594 vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3597 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3599 /* verify speculative enqueue, maybe switch current next frame */
3600 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3601 to_next, n_left_to_next,
3605 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3608 vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3609 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3611 return frame->n_vectors;
3614 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3615 .function = snat_hairpin_dst_fn,
3616 .name = "nat44-hairpin-dst",
3617 .vector_size = sizeof (u32),
3618 .type = VLIB_NODE_TYPE_INTERNAL,
3619 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3620 .error_strings = snat_in2out_error_strings,
3623 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3624 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3628 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3629 snat_hairpin_dst_fn);
3632 snat_hairpin_src_fn (vlib_main_t * vm,
3633 vlib_node_runtime_t * node,
3634 vlib_frame_t * frame)
3636 u32 n_left_from, * from, * to_next;
3637 snat_in2out_next_t next_index;
3638 u32 pkts_processed = 0;
3639 snat_main_t *sm = &snat_main;
3641 from = vlib_frame_vector_args (frame);
3642 n_left_from = frame->n_vectors;
3643 next_index = node->cached_next_index;
3645 while (n_left_from > 0)
3649 vlib_get_next_frame (vm, node, next_index,
3650 to_next, n_left_to_next);
3652 while (n_left_from > 0 && n_left_to_next > 0)
3657 snat_interface_t *i;
3660 /* speculatively enqueue b0 to the current next frame */
3666 n_left_to_next -= 1;
3668 b0 = vlib_get_buffer (vm, bi0);
3669 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3670 next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3672 pool_foreach (i, sm->output_feature_interfaces,
3674 /* Only packets from NAT inside interface */
3675 if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3677 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3678 SNAT_FLAG_HAIRPINNING))
3680 if (PREDICT_TRUE (sm->num_workers > 1))
3681 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3683 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3689 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3691 /* verify speculative enqueue, maybe switch current next frame */
3692 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3693 to_next, n_left_to_next,
3697 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3700 vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3701 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3703 return frame->n_vectors;
3706 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3707 .function = snat_hairpin_src_fn,
3708 .name = "nat44-hairpin-src",
3709 .vector_size = sizeof (u32),
3710 .type = VLIB_NODE_TYPE_INTERNAL,
3711 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3712 .error_strings = snat_in2out_error_strings,
3713 .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3715 [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3716 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3717 [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3718 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3722 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3723 snat_hairpin_src_fn);
3726 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3727 vlib_node_runtime_t * node,
3728 vlib_frame_t * frame)
3730 u32 n_left_from, * from, * to_next;
3731 snat_in2out_next_t next_index;
3732 u32 pkts_processed = 0;
3733 snat_main_t * sm = &snat_main;
3734 u32 stats_node_index;
3736 stats_node_index = snat_in2out_fast_node.index;
3738 from = vlib_frame_vector_args (frame);
3739 n_left_from = frame->n_vectors;
3740 next_index = node->cached_next_index;
3742 while (n_left_from > 0)
3746 vlib_get_next_frame (vm, node, next_index,
3747 to_next, n_left_to_next);
3749 while (n_left_from > 0 && n_left_to_next > 0)
3757 u32 new_addr0, old_addr0;
3758 u16 old_port0, new_port0;
3759 udp_header_t * udp0;
3760 tcp_header_t * tcp0;
3761 icmp46_header_t * icmp0;
3762 snat_session_key_t key0, sm0;
3766 /* speculatively enqueue b0 to the current next frame */
3772 n_left_to_next -= 1;
3774 b0 = vlib_get_buffer (vm, bi0);
3775 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3777 ip0 = vlib_buffer_get_current (b0);
3778 udp0 = ip4_next_header (ip0);
3779 tcp0 = (tcp_header_t *) udp0;
3780 icmp0 = (icmp46_header_t *) udp0;
3782 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3783 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3785 if (PREDICT_FALSE(ip0->ttl == 1))
3787 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3788 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3789 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3791 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3795 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3797 if (PREDICT_FALSE (proto0 == ~0))
3800 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3802 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3803 rx_fib_index0, node, next0, ~0, 0, 0);
3807 key0.addr = ip0->src_address;
3808 key0.protocol = proto0;
3809 key0.port = udp0->src_port;
3810 key0.fib_index = rx_fib_index0;
3812 if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0))
3814 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3815 next0= SNAT_IN2OUT_NEXT_DROP;
3819 new_addr0 = sm0.addr.as_u32;
3820 new_port0 = sm0.port;
3821 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3822 old_addr0 = ip0->src_address.as_u32;
3823 ip0->src_address.as_u32 = new_addr0;
3825 sum0 = ip0->checksum;
3826 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3828 src_address /* changed member */);
3829 ip0->checksum = ip_csum_fold (sum0);
3831 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3833 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3835 old_port0 = tcp0->src_port;
3836 tcp0->src_port = new_port0;
3838 sum0 = tcp0->checksum;
3839 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3841 dst_address /* changed member */);
3842 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3843 ip4_header_t /* cheat */,
3844 length /* changed member */);
3845 tcp0->checksum = ip_csum_fold(sum0);
3849 old_port0 = udp0->src_port;
3850 udp0->src_port = new_port0;
3856 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3858 sum0 = tcp0->checksum;
3859 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3861 dst_address /* changed member */);
3862 tcp0->checksum = ip_csum_fold(sum0);
3867 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3870 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3871 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3873 snat_in2out_trace_t *t =
3874 vlib_add_trace (vm, node, b0, sizeof (*t));
3875 t->sw_if_index = sw_if_index0;
3876 t->next_index = next0;
3879 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3881 /* verify speculative enqueue, maybe switch current next frame */
3882 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3883 to_next, n_left_to_next,
3887 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3890 vlib_node_increment_counter (vm, stats_node_index,
3891 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3893 return frame->n_vectors;
3897 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
3898 .function = snat_in2out_fast_static_map_fn,
3899 .name = "nat44-in2out-fast",
3900 .vector_size = sizeof (u32),
3901 .format_trace = format_snat_in2out_fast_trace,
3902 .type = VLIB_NODE_TYPE_INTERNAL,
3904 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3905 .error_strings = snat_in2out_error_strings,
3907 .runtime_data_bytes = sizeof (snat_runtime_t),
3909 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3911 /* edit / add dispositions here */
3913 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3914 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3915 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
3916 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3917 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3921 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);