2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/fib/ip4_fib.h>
25 #include <nat/nat_ipfix_logging.h>
26 #include <nat/nat_det.h>
27 #include <nat/nat_reass.h>
28 #include <nat/nat_inlines.h>
30 #include <vppinfra/hash.h>
31 #include <vppinfra/error.h>
32 #include <vppinfra/elog.h>
39 } snat_in2out_trace_t;
42 u32 next_worker_index;
44 } snat_in2out_worker_handoff_trace_t;
46 /* packet trace format function */
47 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
49 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
50 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
51 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
54 tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
56 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
57 t->sw_if_index, t->next_index, t->session_index);
62 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
64 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
65 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
66 snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
68 s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
69 t->sw_if_index, t->next_index);
74 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
76 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
77 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
78 snat_in2out_worker_handoff_trace_t * t =
79 va_arg (*args, snat_in2out_worker_handoff_trace_t *);
82 m = t->do_handoff ? "next worker" : "same worker";
83 s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
92 } nat44_in2out_reass_trace_t;
94 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
96 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
97 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
98 nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
100 s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
101 t->sw_if_index, t->next_index,
102 t->cached ? "cached" : "translated");
107 vlib_node_registration_t snat_in2out_node;
108 vlib_node_registration_t snat_in2out_slowpath_node;
109 vlib_node_registration_t snat_in2out_fast_node;
110 vlib_node_registration_t snat_in2out_worker_handoff_node;
111 vlib_node_registration_t snat_det_in2out_node;
112 vlib_node_registration_t snat_in2out_output_node;
113 vlib_node_registration_t snat_in2out_output_slowpath_node;
114 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
115 vlib_node_registration_t snat_hairpin_dst_node;
116 vlib_node_registration_t snat_hairpin_src_node;
117 vlib_node_registration_t nat44_hairpinning_node;
118 vlib_node_registration_t nat44_in2out_reass_node;
119 vlib_node_registration_t nat44_ed_in2out_node;
120 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
121 vlib_node_registration_t nat44_ed_in2out_output_node;
122 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
123 vlib_node_registration_t nat44_ed_hairpin_dst_node;
124 vlib_node_registration_t nat44_ed_hairpin_src_node;
125 vlib_node_registration_t nat44_ed_hairpinning_node;
127 #define foreach_snat_in2out_error \
128 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
129 _(IN2OUT_PACKETS, "Good in2out packets processed") \
130 _(OUT_OF_PORTS, "Out of ports") \
131 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found") \
132 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
133 _(NO_TRANSLATION, "No translation") \
134 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
135 _(DROP_FRAGMENT, "Drop fragment") \
136 _(MAX_REASS, "Maximum reassemblies exceeded") \
137 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
138 _(FQ_CONGESTED, "Handoff frame queue congested")
141 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
142 foreach_snat_in2out_error
145 } snat_in2out_error_t;
147 static char * snat_in2out_error_strings[] = {
148 #define _(sym,string) string,
149 foreach_snat_in2out_error
154 SNAT_IN2OUT_NEXT_LOOKUP,
155 SNAT_IN2OUT_NEXT_DROP,
156 SNAT_IN2OUT_NEXT_ICMP_ERROR,
157 SNAT_IN2OUT_NEXT_SLOW_PATH,
158 SNAT_IN2OUT_NEXT_REASS,
160 } snat_in2out_next_t;
163 SNAT_HAIRPIN_SRC_NEXT_DROP,
164 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
165 SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
166 SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
167 SNAT_HAIRPIN_SRC_N_NEXT,
168 } snat_hairpin_next_t;
171 * @brief Check if packet should be translated
173 * Packets aimed at outside interface and external address with active session
174 * should be translated.
177 * @param rt NAT runtime data
178 * @param sw_if_index0 index of the inside interface
179 * @param ip0 IPv4 header
180 * @param proto0 NAT protocol
181 * @param rx_fib_index0 RX FIB index
183 * @returns 0 if packet should be translated otherwise 1
186 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
187 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
193 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
194 nat_outside_fib_t *outside_fib;
196 .fp_proto = FIB_PROTOCOL_IP4,
199 .ip4.as_u32 = ip0->dst_address.as_u32,
203 /* Don't NAT packet aimed at the intfc address */
204 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
205 ip0->dst_address.as_u32)))
208 fei = fib_table_lookup (rx_fib_index0, &pfx);
209 if (FIB_NODE_INDEX_INVALID != fei)
211 u32 sw_if_index = fib_entry_get_resolving_interface (fei);
212 if (sw_if_index == ~0)
214 vec_foreach (outside_fib, sm->outside_fibs)
216 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
217 if (FIB_NODE_INDEX_INVALID != fei)
219 sw_if_index = fib_entry_get_resolving_interface (fei);
220 if (sw_if_index != ~0)
225 if (sw_if_index == ~0)
229 pool_foreach (i, sm->interfaces,
231 /* NAT packet aimed at outside interface */
232 if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
241 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
242 u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
243 u32 rx_fib_index0, u32 thread_index)
245 udp_header_t * udp0 = ip4_next_header (ip0);
246 snat_session_key_t key0, sm0;
247 clib_bihash_kv_8_8_t kv0, value0;
249 key0.addr = ip0->dst_address;
250 key0.port = udp0->dst_port;
251 key0.protocol = proto0;
252 key0.fib_index = sm->outside_fib_index;
253 kv0.key = key0.as_u64;
255 /* NAT packet aimed at external address if */
256 /* has active sessions */
257 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
260 /* or is static mappings */
261 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0, 0))
267 if (sm->forwarding_enabled)
270 return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
275 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
276 u32 proto0, u16 src_port, u16 dst_port,
277 u32 thread_index, u32 sw_if_index)
279 snat_session_key_t key0;
280 clib_bihash_kv_8_8_t kv0, value0;
284 key0.addr = ip0->src_address;
285 key0.port = src_port;
286 key0.protocol = proto0;
287 key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
288 kv0.key = key0.as_u64;
290 if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
295 key0.addr = ip0->dst_address;
296 key0.port = dst_port;
297 key0.protocol = proto0;
298 kv0.key = key0.as_u64;
299 if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
303 pool_foreach (i, sm->output_feature_interfaces,
305 if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
315 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void * arg)
317 snat_main_t *sm = &snat_main;
318 nat44_is_idle_session_ctx_t *ctx = arg;
320 u64 sess_timeout_time;
321 snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
323 clib_bihash_kv_8_8_t s_kv;
325 s = pool_elt_at_index (tsm->sessions, kv->value);
326 sess_timeout_time = s->last_heard + (f64)nat44_session_get_timeout(sm, s);
327 if (ctx->now >= sess_timeout_time)
329 s_kv.key = s->out2in.as_u64;
330 if (clib_bihash_add_del_8_8 (&tsm->out2in, &s_kv, 0))
331 nat_log_warn ("out2in key del failed");
333 snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
334 s->out2in.addr.as_u32,
338 s->in2out.fib_index);
340 if (!snat_is_session_static (s))
341 snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
344 nat44_delete_session (sm, s, ctx->thread_index);
351 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
354 snat_session_key_t * key0,
355 snat_session_t ** sessionp,
356 vlib_node_runtime_t * node,
363 clib_bihash_kv_8_8_t kv0;
364 snat_session_key_t key1;
365 u32 address_index = ~0;
366 udp_header_t * udp0 = ip4_next_header (ip0);
368 nat_outside_fib_t *outside_fib;
369 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
371 .fp_proto = FIB_PROTOCOL_IP4,
374 .ip4.as_u32 = ip0->dst_address.as_u32,
377 nat44_is_idle_session_ctx_t ctx0;
379 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
381 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
382 nat_ipfix_logging_max_sessions(sm->max_translations);
383 nat_log_notice ("maximum sessions exceeded");
384 return SNAT_IN2OUT_NEXT_DROP;
387 key1.protocol = key0->protocol;
389 /* First try to match static mapping by local address and port */
390 if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0, 0))
392 /* Try to create dynamic translation */
393 if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
397 sm->per_thread_data[thread_index].snat_thread_index))
399 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
400 return SNAT_IN2OUT_NEXT_DROP;
406 u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
410 nat_log_warn ("create NAT user failed");
411 return SNAT_IN2OUT_NEXT_DROP;
414 s = nat_session_alloc_or_recycle (sm, u, thread_index);
417 nat44_delete_user_with_no_session (sm, u, thread_index);
418 nat_log_warn ("create NAT session failed");
419 return SNAT_IN2OUT_NEXT_DROP;
423 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
424 user_session_increment (sm, u, is_sm);
425 s->outside_address_index = address_index;
428 s->out2in.protocol = key0->protocol;
429 s->out2in.fib_index = sm->outside_fib_index;
430 switch (vec_len (sm->outside_fibs))
433 s->out2in.fib_index = sm->outside_fib_index;
436 s->out2in.fib_index = sm->outside_fibs[0].fib_index;
439 vec_foreach (outside_fib, sm->outside_fibs)
441 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
442 if (FIB_NODE_INDEX_INVALID != fei)
444 if (fib_entry_get_resolving_interface (fei) != ~0)
446 s->out2in.fib_index = outside_fib->fib_index;
453 s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
454 s->ext_host_port = udp0->dst_port;
457 /* Add to translation hashes */
459 ctx0.thread_index = thread_index;
460 kv0.key = s->in2out.as_u64;
461 kv0.value = s - sm->per_thread_data[thread_index].sessions;
462 if (clib_bihash_add_or_overwrite_stale_8_8 (
463 &sm->per_thread_data[thread_index].in2out, &kv0,
464 nat44_i2o_is_idle_session_cb, &ctx0))
465 nat_log_notice ("in2out key add failed");
467 kv0.key = s->out2in.as_u64;
468 kv0.value = s - sm->per_thread_data[thread_index].sessions;
470 if (clib_bihash_add_or_overwrite_stale_8_8 (
471 &sm->per_thread_data[thread_index].out2in, &kv0,
472 nat44_o2i_is_idle_session_cb, &ctx0))
473 nat_log_notice ("out2in key add failed");
476 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
477 s->out2in.addr.as_u32,
481 s->in2out.fib_index);
486 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
487 snat_session_key_t *p_key0)
489 icmp46_header_t *icmp0;
490 snat_session_key_t key0;
491 icmp_echo_header_t *echo0, *inner_echo0 = 0;
492 ip4_header_t *inner_ip0 = 0;
494 icmp46_header_t *inner_icmp0;
496 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
497 echo0 = (icmp_echo_header_t *)(icmp0+1);
499 if (!icmp_is_error_message (icmp0))
501 key0.protocol = SNAT_PROTOCOL_ICMP;
502 key0.addr = ip0->src_address;
503 key0.port = echo0->identifier;
507 inner_ip0 = (ip4_header_t *)(echo0+1);
508 l4_header = ip4_next_header (inner_ip0);
509 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
510 key0.addr = inner_ip0->dst_address;
511 switch (key0.protocol)
513 case SNAT_PROTOCOL_ICMP:
514 inner_icmp0 = (icmp46_header_t*)l4_header;
515 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
516 key0.port = inner_echo0->identifier;
518 case SNAT_PROTOCOL_UDP:
519 case SNAT_PROTOCOL_TCP:
520 key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
523 return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
527 return -1; /* success */
531 * Get address and port values to be used for ICMP packet translation
532 * and create session if needed
534 * @param[in,out] sm NAT main
535 * @param[in,out] node NAT node runtime
536 * @param[in] thread_index thread index
537 * @param[in,out] b0 buffer containing packet to be translated
538 * @param[out] p_proto protocol used for matching
539 * @param[out] p_value address and port after NAT translation
540 * @param[out] p_dont_translate if packet should not be translated
541 * @param d optional parameter
542 * @param e optional parameter
544 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
545 u32 thread_index, vlib_buffer_t *b0,
546 ip4_header_t *ip0, u8 *p_proto,
547 snat_session_key_t *p_value,
548 u8 *p_dont_translate, void *d, void *e)
550 icmp46_header_t *icmp0;
553 snat_session_key_t key0;
554 snat_session_t *s0 = 0;
555 u8 dont_translate = 0;
556 clib_bihash_kv_8_8_t kv0, value0;
560 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
561 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
562 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
564 err = icmp_get_key (ip0, &key0);
567 b0->error = node->errors[err];
568 next0 = SNAT_IN2OUT_NEXT_DROP;
571 key0.fib_index = rx_fib_index0;
573 kv0.key = key0.as_u64;
575 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
578 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
580 if (PREDICT_FALSE(nat_not_translate_output_feature(sm, ip0,
581 key0.protocol, key0.port, key0.port, thread_index, sw_if_index0)))
589 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
590 ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
597 if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
599 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
600 next0 = SNAT_IN2OUT_NEXT_DROP;
604 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, &s0, node, next0,
605 thread_index, vlib_time_now (sm->vlib_main));
607 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
612 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
613 icmp0->type != ICMP4_echo_reply &&
614 !icmp_is_error_message (icmp0)))
616 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
617 next0 = SNAT_IN2OUT_NEXT_DROP;
621 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
626 *p_proto = key0.protocol;
628 *p_value = s0->out2in;
629 *p_dont_translate = dont_translate;
631 *(snat_session_t**)d = s0;
636 * Get address and port values to be used for ICMP packet translation
638 * @param[in] sm NAT main
639 * @param[in,out] node NAT node runtime
640 * @param[in] thread_index thread index
641 * @param[in,out] b0 buffer containing packet to be translated
642 * @param[out] p_proto protocol used for matching
643 * @param[out] p_value address and port after NAT translation
644 * @param[out] p_dont_translate if packet should not be translated
645 * @param d optional parameter
646 * @param e optional parameter
648 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
649 u32 thread_index, vlib_buffer_t *b0,
650 ip4_header_t *ip0, u8 *p_proto,
651 snat_session_key_t *p_value,
652 u8 *p_dont_translate, void *d, void *e)
654 icmp46_header_t *icmp0;
657 snat_session_key_t key0;
658 snat_session_key_t sm0;
659 u8 dont_translate = 0;
664 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
665 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
666 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
668 err = icmp_get_key (ip0, &key0);
671 b0->error = node->errors[err];
672 next0 = SNAT_IN2OUT_NEXT_DROP;
675 key0.fib_index = rx_fib_index0;
677 if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0, 0))
679 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
680 IP_PROTOCOL_ICMP, rx_fib_index0)))
686 if (icmp_is_error_message (icmp0))
688 next0 = SNAT_IN2OUT_NEXT_DROP;
692 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
693 next0 = SNAT_IN2OUT_NEXT_DROP;
697 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
698 (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
699 !icmp_is_error_message (icmp0)))
701 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
702 next0 = SNAT_IN2OUT_NEXT_DROP;
709 *p_proto = key0.protocol;
710 *p_dont_translate = dont_translate;
714 static inline u32 icmp_in2out (snat_main_t *sm,
717 icmp46_header_t * icmp0,
720 vlib_node_runtime_t * node,
726 snat_session_key_t sm0;
728 icmp_echo_header_t *echo0, *inner_echo0 = 0;
729 ip4_header_t *inner_ip0;
731 icmp46_header_t *inner_icmp0;
733 u32 new_addr0, old_addr0;
734 u16 old_id0, new_id0;
739 echo0 = (icmp_echo_header_t *)(icmp0+1);
741 next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
742 &protocol, &sm0, &dont_translate, d, e);
745 if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
748 sum0 = ip_incremental_checksum (0, icmp0,
749 ntohs(ip0->length) - ip4_header_bytes (ip0));
750 checksum0 = ~ip_csum_fold (sum0);
751 if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
753 next0 = SNAT_IN2OUT_NEXT_DROP;
757 old_addr0 = ip0->src_address.as_u32;
758 new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
759 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
760 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
762 sum0 = ip0->checksum;
763 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
764 src_address /* changed member */);
765 ip0->checksum = ip_csum_fold (sum0);
767 if (icmp0->checksum == 0)
768 icmp0->checksum = 0xffff;
770 if (!icmp_is_error_message (icmp0))
773 if (PREDICT_FALSE(new_id0 != echo0->identifier))
775 old_id0 = echo0->identifier;
777 echo0->identifier = new_id0;
779 sum0 = icmp0->checksum;
780 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
782 icmp0->checksum = ip_csum_fold (sum0);
787 inner_ip0 = (ip4_header_t *)(echo0+1);
788 l4_header = ip4_next_header (inner_ip0);
790 if (!ip4_header_checksum_is_valid (inner_ip0))
792 next0 = SNAT_IN2OUT_NEXT_DROP;
796 old_addr0 = inner_ip0->dst_address.as_u32;
797 inner_ip0->dst_address = sm0.addr;
798 new_addr0 = inner_ip0->dst_address.as_u32;
800 sum0 = icmp0->checksum;
801 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
802 dst_address /* changed member */);
803 icmp0->checksum = ip_csum_fold (sum0);
807 case SNAT_PROTOCOL_ICMP:
808 inner_icmp0 = (icmp46_header_t*)l4_header;
809 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
811 old_id0 = inner_echo0->identifier;
813 inner_echo0->identifier = new_id0;
815 sum0 = icmp0->checksum;
816 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
818 icmp0->checksum = ip_csum_fold (sum0);
820 case SNAT_PROTOCOL_UDP:
821 case SNAT_PROTOCOL_TCP:
822 old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
824 ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
826 sum0 = icmp0->checksum;
827 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
829 icmp0->checksum = ip_csum_fold (sum0);
843 * Hairpinning allows two endpoints on the internal side of the NAT to
844 * communicate even if they only use each other's external IP addresses
847 * @param sm NAT main.
848 * @param b0 Vlib buffer.
849 * @param ip0 IP header.
850 * @param udp0 UDP header.
851 * @param tcp0 TCP header.
852 * @param proto0 NAT protocol.
855 snat_hairpinning (snat_main_t *sm,
863 snat_session_key_t key0, sm0;
865 clib_bihash_kv_8_8_t kv0, value0;
867 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
868 u16 new_dst_port0, old_dst_port0;
871 key0.addr = ip0->dst_address;
872 key0.port = udp0->dst_port;
873 key0.protocol = proto0;
874 key0.fib_index = sm->outside_fib_index;
875 kv0.key = key0.as_u64;
877 /* Check if destination is static mappings */
878 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0, 0))
880 new_dst_addr0 = sm0.addr.as_u32;
881 new_dst_port0 = sm0.port;
882 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
884 /* or active session */
887 if (sm->num_workers > 1)
888 ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
890 ti = sm->num_workers;
894 clib_bihash_kv_16_8_t ed_kv, ed_value;
895 make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
896 ip0->protocol, sm->outside_fib_index, udp0->dst_port,
898 rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
904 rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
911 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
912 new_dst_addr0 = s0->in2out.addr.as_u32;
913 new_dst_port0 = s0->in2out.port;
914 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
917 /* Destination is behind the same NAT, use internal address and port */
920 old_dst_addr0 = ip0->dst_address.as_u32;
921 ip0->dst_address.as_u32 = new_dst_addr0;
922 sum0 = ip0->checksum;
923 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
924 ip4_header_t, dst_address);
925 ip0->checksum = ip_csum_fold (sum0);
927 old_dst_port0 = tcp0->dst;
928 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
930 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
932 tcp0->dst = new_dst_port0;
933 sum0 = tcp0->checksum;
934 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
935 ip4_header_t, dst_address);
936 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
937 ip4_header_t /* cheat */, length);
938 tcp0->checksum = ip_csum_fold(sum0);
942 udp0->dst_port = new_dst_port0;
948 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
950 sum0 = tcp0->checksum;
951 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
952 ip4_header_t, dst_address);
953 tcp0->checksum = ip_csum_fold(sum0);
962 snat_icmp_hairpinning (snat_main_t *sm,
965 icmp46_header_t * icmp0,
968 snat_session_key_t key0, sm0;
969 clib_bihash_kv_8_8_t kv0, value0;
970 u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
975 if (!icmp_is_error_message (icmp0))
977 icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
978 u16 icmp_id0 = echo0->identifier;
979 key0.addr = ip0->dst_address;
980 key0.port = icmp_id0;
981 key0.protocol = SNAT_PROTOCOL_ICMP;
982 key0.fib_index = sm->outside_fib_index;
983 kv0.key = key0.as_u64;
985 if (sm->num_workers > 1)
986 ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
988 ti = sm->num_workers;
990 /* Check if destination is in active sessions */
993 clib_bihash_kv_16_8_t ed_kv, ed_value;
994 make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
995 IP_PROTOCOL_ICMP, sm->outside_fib_index, icmp_id0, 0);
996 rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
1002 rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1008 /* or static mappings */
1009 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0, 0))
1011 new_dst_addr0 = sm0.addr.as_u32;
1012 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1017 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1018 new_dst_addr0 = s0->in2out.addr.as_u32;
1019 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1020 echo0->identifier = s0->in2out.port;
1021 sum0 = icmp0->checksum;
1022 sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1023 icmp_echo_header_t, identifier);
1024 icmp0->checksum = ip_csum_fold (sum0);
1027 /* Destination is behind the same NAT, use internal address and port */
1030 old_dst_addr0 = ip0->dst_address.as_u32;
1031 ip0->dst_address.as_u32 = new_dst_addr0;
1032 sum0 = ip0->checksum;
1033 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1034 ip4_header_t, dst_address);
1035 ip0->checksum = ip_csum_fold (sum0);
1041 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1044 icmp46_header_t * icmp0,
1047 vlib_node_runtime_t * node,
1051 snat_session_t ** p_s0)
1053 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1054 next0, thread_index, p_s0, 0);
1055 snat_session_t * s0 = *p_s0;
1056 if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1059 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1060 snat_icmp_hairpinning(sm, b0, ip0, icmp0, sm->endpoint_dependent);
1062 nat44_session_update_counters (s0, now,
1063 vlib_buffer_length_in_chain (sm->vlib_main, b0));
1064 /* Per-user LRU list maintenance */
1065 nat44_session_update_lru (sm, s0, thread_index);
1071 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
1075 clib_bihash_kv_8_8_t kv, value;
1076 snat_static_mapping_t *m;
1077 u32 old_addr, new_addr;
1080 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1081 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1084 m = pool_elt_at_index (sm->static_mappings, value.value);
1086 old_addr = ip->dst_address.as_u32;
1087 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1089 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1090 ip->checksum = ip_csum_fold (sum);
1092 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1093 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1097 nat_in2out_sm_unknown_proto (snat_main_t *sm,
1102 clib_bihash_kv_8_8_t kv, value;
1103 snat_static_mapping_t *m;
1104 snat_session_key_t m_key;
1105 u32 old_addr, new_addr;
1108 m_key.addr = ip->src_address;
1111 m_key.fib_index = rx_fib_index;
1112 kv.key = m_key.as_u64;
1113 if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1116 m = pool_elt_at_index (sm->static_mappings, value.value);
1118 old_addr = ip->src_address.as_u32;
1119 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1121 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1122 ip->checksum = ip_csum_fold (sum);
1126 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1128 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1129 nat_hairpinning_sm_unknown_proto (sm, b, ip);
1136 snat_in2out_node_fn_inline (vlib_main_t * vm,
1137 vlib_node_runtime_t * node,
1138 vlib_frame_t * frame, int is_slow_path,
1139 int is_output_feature)
1141 u32 n_left_from, * from, * to_next;
1142 snat_in2out_next_t next_index;
1143 u32 pkts_processed = 0;
1144 snat_main_t * sm = &snat_main;
1145 f64 now = vlib_time_now (vm);
1146 u32 stats_node_index;
1147 u32 thread_index = vm->thread_index;
1149 stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1150 snat_in2out_node.index;
1152 from = vlib_frame_vector_args (frame);
1153 n_left_from = frame->n_vectors;
1154 next_index = node->cached_next_index;
1156 while (n_left_from > 0)
1160 vlib_get_next_frame (vm, node, next_index,
1161 to_next, n_left_to_next);
1163 while (n_left_from >= 4 && n_left_to_next >= 2)
1166 vlib_buffer_t * b0, * b1;
1168 u32 sw_if_index0, sw_if_index1;
1169 ip4_header_t * ip0, * ip1;
1170 ip_csum_t sum0, sum1;
1171 u32 new_addr0, old_addr0, new_addr1, old_addr1;
1172 u16 old_port0, new_port0, old_port1, new_port1;
1173 udp_header_t * udp0, * udp1;
1174 tcp_header_t * tcp0, * tcp1;
1175 icmp46_header_t * icmp0, * icmp1;
1176 snat_session_key_t key0, key1;
1177 u32 rx_fib_index0, rx_fib_index1;
1179 snat_session_t * s0 = 0, * s1 = 0;
1180 clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1181 u32 iph_offset0 = 0, iph_offset1 = 0;
1183 /* Prefetch next iteration. */
1185 vlib_buffer_t * p2, * p3;
1187 p2 = vlib_get_buffer (vm, from[2]);
1188 p3 = vlib_get_buffer (vm, from[3]);
1190 vlib_prefetch_buffer_header (p2, LOAD);
1191 vlib_prefetch_buffer_header (p3, LOAD);
1193 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1194 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1197 /* speculatively enqueue b0 and b1 to the current next frame */
1198 to_next[0] = bi0 = from[0];
1199 to_next[1] = bi1 = from[1];
1203 n_left_to_next -= 2;
1205 b0 = vlib_get_buffer (vm, bi0);
1206 b1 = vlib_get_buffer (vm, bi1);
1208 if (is_output_feature)
1209 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1211 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1214 udp0 = ip4_next_header (ip0);
1215 tcp0 = (tcp_header_t *) udp0;
1216 icmp0 = (icmp46_header_t *) udp0;
1218 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1219 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1222 next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1224 if (PREDICT_FALSE(ip0->ttl == 1))
1226 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1227 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1228 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1230 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1234 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1236 /* Next configured feature, probably ip4-lookup */
1239 if (PREDICT_FALSE (proto0 == ~0))
1241 if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1243 next0 = SNAT_IN2OUT_NEXT_DROP;
1244 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1249 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1251 next0 = icmp_in2out_slow_path
1252 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1253 node, next0, now, thread_index, &s0);
1259 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1261 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1265 if (ip4_is_fragment (ip0))
1267 next0 = SNAT_IN2OUT_NEXT_REASS;
1272 key0.addr = ip0->src_address;
1273 key0.port = udp0->src_port;
1274 key0.protocol = proto0;
1275 key0.fib_index = rx_fib_index0;
1277 kv0.key = key0.as_u64;
1279 if (PREDICT_FALSE (clib_bihash_search_8_8 (
1280 &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1284 if (is_output_feature)
1286 if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1287 ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1292 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1293 ip0, proto0, rx_fib_index0, thread_index)))
1297 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1298 &s0, node, next0, thread_index, now);
1299 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1304 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1309 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1312 b0->flags |= VNET_BUFFER_F_IS_NATED;
1314 old_addr0 = ip0->src_address.as_u32;
1315 ip0->src_address = s0->out2in.addr;
1316 new_addr0 = ip0->src_address.as_u32;
1317 if (!is_output_feature)
1318 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1320 sum0 = ip0->checksum;
1321 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1323 src_address /* changed member */);
1324 ip0->checksum = ip_csum_fold (sum0);
1326 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1328 old_port0 = tcp0->src_port;
1329 tcp0->src_port = s0->out2in.port;
1330 new_port0 = tcp0->src_port;
1332 sum0 = tcp0->checksum;
1333 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1335 dst_address /* changed member */);
1336 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1337 ip4_header_t /* cheat */,
1338 length /* changed member */);
1339 mss_clamping (sm, tcp0, &sum0);
1340 tcp0->checksum = ip_csum_fold(sum0);
1344 old_port0 = udp0->src_port;
1345 udp0->src_port = s0->out2in.port;
1350 nat44_session_update_counters (s0, now,
1351 vlib_buffer_length_in_chain (vm, b0));
1352 /* Per-user LRU list maintenance */
1353 nat44_session_update_lru (sm, s0, thread_index);
1356 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1357 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1359 snat_in2out_trace_t *t =
1360 vlib_add_trace (vm, node, b0, sizeof (*t));
1361 t->is_slow_path = is_slow_path;
1362 t->sw_if_index = sw_if_index0;
1363 t->next_index = next0;
1364 t->session_index = ~0;
1366 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1369 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1371 if (is_output_feature)
1372 iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1374 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1377 udp1 = ip4_next_header (ip1);
1378 tcp1 = (tcp_header_t *) udp1;
1379 icmp1 = (icmp46_header_t *) udp1;
1381 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1382 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1385 if (PREDICT_FALSE(ip1->ttl == 1))
1387 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1388 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1389 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1391 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1395 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1397 /* Next configured feature, probably ip4-lookup */
1400 if (PREDICT_FALSE (proto1 == ~0))
1402 if (nat_in2out_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1404 next1 = SNAT_IN2OUT_NEXT_DROP;
1405 b1->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1410 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1412 next1 = icmp_in2out_slow_path
1413 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1414 next1, now, thread_index, &s1);
1420 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1422 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1426 if (ip4_is_fragment (ip1))
1428 next1 = SNAT_IN2OUT_NEXT_REASS;
1433 key1.addr = ip1->src_address;
1434 key1.port = udp1->src_port;
1435 key1.protocol = proto1;
1436 key1.fib_index = rx_fib_index1;
1438 kv1.key = key1.as_u64;
1440 if (PREDICT_FALSE(clib_bihash_search_8_8 (
1441 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1445 if (is_output_feature)
1447 if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1448 ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1453 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1454 ip1, proto1, rx_fib_index1, thread_index)))
1458 next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1459 &s1, node, next1, thread_index, now);
1460 if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1465 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1470 s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1473 b1->flags |= VNET_BUFFER_F_IS_NATED;
1475 old_addr1 = ip1->src_address.as_u32;
1476 ip1->src_address = s1->out2in.addr;
1477 new_addr1 = ip1->src_address.as_u32;
1478 if (!is_output_feature)
1479 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1481 sum1 = ip1->checksum;
1482 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1484 src_address /* changed member */);
1485 ip1->checksum = ip_csum_fold (sum1);
1487 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1489 old_port1 = tcp1->src_port;
1490 tcp1->src_port = s1->out2in.port;
1491 new_port1 = tcp1->src_port;
1493 sum1 = tcp1->checksum;
1494 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1496 dst_address /* changed member */);
1497 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1498 ip4_header_t /* cheat */,
1499 length /* changed member */);
1500 mss_clamping (sm, tcp1, &sum1);
1501 tcp1->checksum = ip_csum_fold(sum1);
1505 old_port1 = udp1->src_port;
1506 udp1->src_port = s1->out2in.port;
1511 nat44_session_update_counters (s1, now,
1512 vlib_buffer_length_in_chain (vm, b1));
1513 /* Per-user LRU list maintenance */
1514 nat44_session_update_lru (sm, s1, thread_index);
1517 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1518 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1520 snat_in2out_trace_t *t =
1521 vlib_add_trace (vm, node, b1, sizeof (*t));
1522 t->sw_if_index = sw_if_index1;
1523 t->next_index = next1;
1524 t->session_index = ~0;
1526 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1529 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1531 /* verify speculative enqueues, maybe switch current next frame */
1532 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1533 to_next, n_left_to_next,
1534 bi0, bi1, next0, next1);
1537 while (n_left_from > 0 && n_left_to_next > 0)
1545 u32 new_addr0, old_addr0;
1546 u16 old_port0, new_port0;
1547 udp_header_t * udp0;
1548 tcp_header_t * tcp0;
1549 icmp46_header_t * icmp0;
1550 snat_session_key_t key0;
1553 snat_session_t * s0 = 0;
1554 clib_bihash_kv_8_8_t kv0, value0;
1555 u32 iph_offset0 = 0;
1557 /* speculatively enqueue b0 to the current next frame */
1563 n_left_to_next -= 1;
1565 b0 = vlib_get_buffer (vm, bi0);
1566 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1568 if (is_output_feature)
1569 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1571 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1574 udp0 = ip4_next_header (ip0);
1575 tcp0 = (tcp_header_t *) udp0;
1576 icmp0 = (icmp46_header_t *) udp0;
1578 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1579 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1582 if (PREDICT_FALSE(ip0->ttl == 1))
1584 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1585 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1586 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1588 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1592 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1594 /* Next configured feature, probably ip4-lookup */
1597 if (PREDICT_FALSE (proto0 == ~0))
1599 if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1601 next0 = SNAT_IN2OUT_NEXT_DROP;
1602 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1607 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1609 next0 = icmp_in2out_slow_path
1610 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1611 next0, now, thread_index, &s0);
1617 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1619 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1623 if (ip4_is_fragment (ip0))
1625 next0 = SNAT_IN2OUT_NEXT_REASS;
1630 key0.addr = ip0->src_address;
1631 key0.port = udp0->src_port;
1632 key0.protocol = proto0;
1633 key0.fib_index = rx_fib_index0;
1635 kv0.key = key0.as_u64;
1637 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1642 if (is_output_feature)
1644 if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1645 ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1650 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1651 ip0, proto0, rx_fib_index0, thread_index)))
1655 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1656 &s0, node, next0, thread_index, now);
1658 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1663 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1668 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1671 b0->flags |= VNET_BUFFER_F_IS_NATED;
1673 old_addr0 = ip0->src_address.as_u32;
1674 ip0->src_address = s0->out2in.addr;
1675 new_addr0 = ip0->src_address.as_u32;
1676 if (!is_output_feature)
1677 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1679 sum0 = ip0->checksum;
1680 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1682 src_address /* changed member */);
1683 ip0->checksum = ip_csum_fold (sum0);
1685 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1687 old_port0 = tcp0->src_port;
1688 tcp0->src_port = s0->out2in.port;
1689 new_port0 = tcp0->src_port;
1691 sum0 = tcp0->checksum;
1692 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1694 dst_address /* changed member */);
1695 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1696 ip4_header_t /* cheat */,
1697 length /* changed member */);
1698 mss_clamping (sm, tcp0, &sum0);
1699 tcp0->checksum = ip_csum_fold(sum0);
1703 old_port0 = udp0->src_port;
1704 udp0->src_port = s0->out2in.port;
1709 nat44_session_update_counters (s0, now,
1710 vlib_buffer_length_in_chain (vm, b0));
1711 /* Per-user LRU list maintenance */
1712 nat44_session_update_lru (sm, s0, thread_index);
1715 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1716 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1718 snat_in2out_trace_t *t =
1719 vlib_add_trace (vm, node, b0, sizeof (*t));
1720 t->is_slow_path = is_slow_path;
1721 t->sw_if_index = sw_if_index0;
1722 t->next_index = next0;
1723 t->session_index = ~0;
1725 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1728 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1730 /* verify speculative enqueue, maybe switch current next frame */
1731 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1732 to_next, n_left_to_next,
1736 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1739 vlib_node_increment_counter (vm, stats_node_index,
1740 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1742 return frame->n_vectors;
1746 snat_in2out_fast_path_fn (vlib_main_t * vm,
1747 vlib_node_runtime_t * node,
1748 vlib_frame_t * frame)
1750 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1753 VLIB_REGISTER_NODE (snat_in2out_node) = {
1754 .function = snat_in2out_fast_path_fn,
1755 .name = "nat44-in2out",
1756 .vector_size = sizeof (u32),
1757 .format_trace = format_snat_in2out_trace,
1758 .type = VLIB_NODE_TYPE_INTERNAL,
1760 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1761 .error_strings = snat_in2out_error_strings,
1763 .runtime_data_bytes = sizeof (snat_runtime_t),
1765 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1767 /* edit / add dispositions here */
1769 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1770 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1771 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1772 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1773 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1777 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1780 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1781 vlib_node_runtime_t * node,
1782 vlib_frame_t * frame)
1784 return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1787 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1788 .function = snat_in2out_output_fast_path_fn,
1789 .name = "nat44-in2out-output",
1790 .vector_size = sizeof (u32),
1791 .format_trace = format_snat_in2out_trace,
1792 .type = VLIB_NODE_TYPE_INTERNAL,
1794 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1795 .error_strings = snat_in2out_error_strings,
1797 .runtime_data_bytes = sizeof (snat_runtime_t),
1799 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1801 /* edit / add dispositions here */
1803 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1804 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1805 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1806 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1807 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1811 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1812 snat_in2out_output_fast_path_fn);
1815 snat_in2out_slow_path_fn (vlib_main_t * vm,
1816 vlib_node_runtime_t * node,
1817 vlib_frame_t * frame)
1819 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
1822 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1823 .function = snat_in2out_slow_path_fn,
1824 .name = "nat44-in2out-slowpath",
1825 .vector_size = sizeof (u32),
1826 .format_trace = format_snat_in2out_trace,
1827 .type = VLIB_NODE_TYPE_INTERNAL,
1829 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1830 .error_strings = snat_in2out_error_strings,
1832 .runtime_data_bytes = sizeof (snat_runtime_t),
1834 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1836 /* edit / add dispositions here */
1838 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1839 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1840 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1841 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1842 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1846 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1847 snat_in2out_slow_path_fn);
1850 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1851 vlib_node_runtime_t * node,
1852 vlib_frame_t * frame)
1854 return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
1857 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1858 .function = snat_in2out_output_slow_path_fn,
1859 .name = "nat44-in2out-output-slowpath",
1860 .vector_size = sizeof (u32),
1861 .format_trace = format_snat_in2out_trace,
1862 .type = VLIB_NODE_TYPE_INTERNAL,
1864 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1865 .error_strings = snat_in2out_error_strings,
1867 .runtime_data_bytes = sizeof (snat_runtime_t),
1869 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1871 /* edit / add dispositions here */
1873 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1874 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1875 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1876 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1877 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1881 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1882 snat_in2out_output_slow_path_fn);
1884 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
1887 nat44_hairpinning_fn_inline (vlib_main_t * vm,
1888 vlib_node_runtime_t * node,
1889 vlib_frame_t * frame,
1892 u32 n_left_from, * from, * to_next, stats_node_index;
1893 snat_in2out_next_t next_index;
1894 u32 pkts_processed = 0;
1895 snat_main_t * sm = &snat_main;
1896 vnet_feature_main_t *fm = &feature_main;
1897 u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1898 vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
1900 stats_node_index = is_ed ? nat44_ed_hairpinning_node.index :
1901 nat44_hairpinning_node.index;
1902 from = vlib_frame_vector_args (frame);
1903 n_left_from = frame->n_vectors;
1904 next_index = node->cached_next_index;
1906 while (n_left_from > 0)
1910 vlib_get_next_frame (vm, node, next_index,
1911 to_next, n_left_to_next);
1913 while (n_left_from > 0 && n_left_to_next > 0)
1920 udp_header_t * udp0;
1921 tcp_header_t * tcp0;
1923 /* speculatively enqueue b0 to the current next frame */
1929 n_left_to_next -= 1;
1931 b0 = vlib_get_buffer (vm, bi0);
1932 ip0 = vlib_buffer_get_current (b0);
1933 udp0 = ip4_next_header (ip0);
1934 tcp0 = (tcp_header_t *) udp0;
1936 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1938 vnet_get_config_data (&cm->config_main, &b0->current_config_index,
1941 if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
1942 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1944 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1946 /* verify speculative enqueue, maybe switch current next frame */
1947 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1948 to_next, n_left_to_next,
1952 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1955 vlib_node_increment_counter (vm, stats_node_index,
1956 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1958 return frame->n_vectors;
1962 nat44_hairpinning_fn (vlib_main_t * vm,
1963 vlib_node_runtime_t * node,
1964 vlib_frame_t * frame)
1966 return nat44_hairpinning_fn_inline (vm, node, frame, 0);
1969 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
1970 .function = nat44_hairpinning_fn,
1971 .name = "nat44-hairpinning",
1972 .vector_size = sizeof (u32),
1973 .type = VLIB_NODE_TYPE_INTERNAL,
1974 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1975 .error_strings = snat_in2out_error_strings,
1978 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1979 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1983 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
1984 nat44_hairpinning_fn);
1987 nat44_ed_hairpinning_fn (vlib_main_t * vm,
1988 vlib_node_runtime_t * node,
1989 vlib_frame_t * frame)
1991 return nat44_hairpinning_fn_inline (vm, node, frame, 1);
1994 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
1995 .function = nat44_ed_hairpinning_fn,
1996 .name = "nat44-ed-hairpinning",
1997 .vector_size = sizeof (u32),
1998 .type = VLIB_NODE_TYPE_INTERNAL,
1999 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2000 .error_strings = snat_in2out_error_strings,
2003 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2004 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2008 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpinning_node,
2009 nat44_ed_hairpinning_fn);
2012 nat44_reass_hairpinning (snat_main_t *sm,
2019 snat_session_key_t key0, sm0;
2020 snat_session_t * s0;
2021 clib_bihash_kv_8_8_t kv0, value0;
2023 u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2024 u16 new_dst_port0, old_dst_port0;
2025 udp_header_t * udp0;
2026 tcp_header_t * tcp0;
2028 key0.addr = ip0->dst_address;
2030 key0.protocol = proto0;
2031 key0.fib_index = sm->outside_fib_index;
2032 kv0.key = key0.as_u64;
2034 udp0 = ip4_next_header (ip0);
2036 /* Check if destination is static mappings */
2037 if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0, 0))
2039 new_dst_addr0 = sm0.addr.as_u32;
2040 new_dst_port0 = sm0.port;
2041 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2043 /* or active sessions */
2046 if (sm->num_workers > 1)
2047 ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2049 ti = sm->num_workers;
2051 if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2054 s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2055 new_dst_addr0 = s0->in2out.addr.as_u32;
2056 new_dst_port0 = s0->in2out.port;
2057 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2061 /* Destination is behind the same NAT, use internal address and port */
2064 old_dst_addr0 = ip0->dst_address.as_u32;
2065 ip0->dst_address.as_u32 = new_dst_addr0;
2066 sum0 = ip0->checksum;
2067 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2068 ip4_header_t, dst_address);
2069 ip0->checksum = ip_csum_fold (sum0);
2071 old_dst_port0 = dport;
2072 if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2073 ip4_is_first_fragment (ip0)))
2075 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2077 tcp0 = ip4_next_header (ip0);
2078 tcp0->dst = new_dst_port0;
2079 sum0 = tcp0->checksum;
2080 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2081 ip4_header_t, dst_address);
2082 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2083 ip4_header_t /* cheat */, length);
2084 tcp0->checksum = ip_csum_fold(sum0);
2088 udp0->dst_port = new_dst_port0;
2094 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2096 tcp0 = ip4_next_header (ip0);
2097 sum0 = tcp0->checksum;
2098 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2099 ip4_header_t, dst_address);
2100 tcp0->checksum = ip_csum_fold(sum0);
2107 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2108 vlib_node_runtime_t * node,
2109 vlib_frame_t * frame)
2111 u32 n_left_from, *from, *to_next;
2112 snat_in2out_next_t next_index;
2113 u32 pkts_processed = 0;
2114 snat_main_t *sm = &snat_main;
2115 f64 now = vlib_time_now (vm);
2116 u32 thread_index = vm->thread_index;
2117 snat_main_per_thread_data_t *per_thread_data =
2118 &sm->per_thread_data[thread_index];
2119 u32 *fragments_to_drop = 0;
2120 u32 *fragments_to_loopback = 0;
2122 from = vlib_frame_vector_args (frame);
2123 n_left_from = frame->n_vectors;
2124 next_index = node->cached_next_index;
2126 while (n_left_from > 0)
2130 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2132 while (n_left_from > 0 && n_left_to_next > 0)
2134 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2139 nat_reass_ip4_t *reass0;
2140 udp_header_t * udp0;
2141 tcp_header_t * tcp0;
2142 snat_session_key_t key0;
2143 clib_bihash_kv_8_8_t kv0, value0;
2144 snat_session_t * s0 = 0;
2145 u16 old_port0, new_port0;
2148 /* speculatively enqueue b0 to the current next frame */
2154 n_left_to_next -= 1;
2156 b0 = vlib_get_buffer (vm, bi0);
2157 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2159 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2160 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2163 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2165 next0 = SNAT_IN2OUT_NEXT_DROP;
2166 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2170 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2171 udp0 = ip4_next_header (ip0);
2172 tcp0 = (tcp_header_t *) udp0;
2173 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2175 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2180 &fragments_to_drop);
2182 if (PREDICT_FALSE (!reass0))
2184 next0 = SNAT_IN2OUT_NEXT_DROP;
2185 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2186 nat_log_notice ("maximum reassemblies exceeded");
2190 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2192 key0.addr = ip0->src_address;
2193 key0.port = udp0->src_port;
2194 key0.protocol = proto0;
2195 key0.fib_index = rx_fib_index0;
2196 kv0.key = key0.as_u64;
2198 if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2200 if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2201 ip0, proto0, rx_fib_index0, thread_index)))
2204 next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2205 &s0, node, next0, thread_index, now);
2207 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2210 reass0->sess_index = s0 - per_thread_data->sessions;
2214 s0 = pool_elt_at_index (per_thread_data->sessions,
2216 reass0->sess_index = value0.value;
2218 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2222 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2224 if (nat_ip4_reass_add_fragment (reass0, bi0, &fragments_to_drop))
2226 b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2227 nat_log_notice ("maximum fragments per reassembly exceeded");
2228 next0 = SNAT_IN2OUT_NEXT_DROP;
2234 s0 = pool_elt_at_index (per_thread_data->sessions,
2235 reass0->sess_index);
2238 old_addr0 = ip0->src_address.as_u32;
2239 ip0->src_address = s0->out2in.addr;
2240 new_addr0 = ip0->src_address.as_u32;
2241 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2243 sum0 = ip0->checksum;
2244 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2246 src_address /* changed member */);
2247 ip0->checksum = ip_csum_fold (sum0);
2249 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2251 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2253 old_port0 = tcp0->src_port;
2254 tcp0->src_port = s0->out2in.port;
2255 new_port0 = tcp0->src_port;
2257 sum0 = tcp0->checksum;
2258 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2260 dst_address /* changed member */);
2261 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2262 ip4_header_t /* cheat */,
2263 length /* changed member */);
2264 tcp0->checksum = ip_csum_fold(sum0);
2268 old_port0 = udp0->src_port;
2269 udp0->src_port = s0->out2in.port;
2275 nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2276 s0->ext_host_port, proto0);
2279 nat44_session_update_counters (s0, now,
2280 vlib_buffer_length_in_chain (vm, b0));
2281 /* Per-user LRU list maintenance */
2282 nat44_session_update_lru (sm, s0, thread_index);
2285 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2286 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2288 nat44_in2out_reass_trace_t *t =
2289 vlib_add_trace (vm, node, b0, sizeof (*t));
2290 t->cached = cached0;
2291 t->sw_if_index = sw_if_index0;
2292 t->next_index = next0;
2302 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2304 /* verify speculative enqueue, maybe switch current next frame */
2305 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2306 to_next, n_left_to_next,
2310 if (n_left_from == 0 && vec_len (fragments_to_loopback))
2312 from = vlib_frame_vector_args (frame);
2313 u32 len = vec_len (fragments_to_loopback);
2314 if (len <= VLIB_FRAME_SIZE)
2316 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2318 vec_reset_length (fragments_to_loopback);
2323 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2324 sizeof (u32) * VLIB_FRAME_SIZE);
2325 n_left_from = VLIB_FRAME_SIZE;
2326 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2331 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2334 vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2335 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2338 nat_send_all_to_node (vm, fragments_to_drop, node,
2339 &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2340 SNAT_IN2OUT_NEXT_DROP);
2342 vec_free (fragments_to_drop);
2343 vec_free (fragments_to_loopback);
2344 return frame->n_vectors;
2347 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2348 .function = nat44_in2out_reass_node_fn,
2349 .name = "nat44-in2out-reass",
2350 .vector_size = sizeof (u32),
2351 .format_trace = format_nat44_in2out_reass_trace,
2352 .type = VLIB_NODE_TYPE_INTERNAL,
2354 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2355 .error_strings = snat_in2out_error_strings,
2357 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2359 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2360 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2361 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2362 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2363 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2367 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2368 nat44_in2out_reass_node_fn);
2370 /*******************************/
2371 /*** endpoint-dependent mode ***/
2372 /*******************************/
2374 static_always_inline int
2375 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
2377 icmp46_header_t *icmp0;
2378 nat_ed_ses_key_t key0;
2379 icmp_echo_header_t *echo0, *inner_echo0 = 0;
2380 ip4_header_t *inner_ip0 = 0;
2381 void *l4_header = 0;
2382 icmp46_header_t *inner_icmp0;
2384 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2385 echo0 = (icmp_echo_header_t *)(icmp0+1);
2387 if (!icmp_is_error_message (icmp0))
2389 key0.proto = IP_PROTOCOL_ICMP;
2390 key0.l_addr = ip0->src_address;
2391 key0.r_addr = ip0->dst_address;
2392 key0.l_port = echo0->identifier;
2397 inner_ip0 = (ip4_header_t *)(echo0+1);
2398 l4_header = ip4_next_header (inner_ip0);
2399 key0.proto = inner_ip0->protocol;
2400 key0.r_addr = inner_ip0->src_address;
2401 key0.l_addr = inner_ip0->dst_address;
2402 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
2404 case SNAT_PROTOCOL_ICMP:
2405 inner_icmp0 = (icmp46_header_t*)l4_header;
2406 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2408 key0.l_port = inner_echo0->identifier;
2410 case SNAT_PROTOCOL_UDP:
2411 case SNAT_PROTOCOL_TCP:
2412 key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2413 key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
2416 return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
2424 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void * arg)
2426 snat_main_t *sm = &snat_main;
2427 nat44_is_idle_session_ctx_t *ctx = arg;
2429 u64 sess_timeout_time;
2430 nat_ed_ses_key_t ed_key;
2431 clib_bihash_kv_16_8_t ed_kv;
2434 snat_session_key_t key;
2435 snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
2438 s = pool_elt_at_index (tsm->sessions, kv->value);
2439 sess_timeout_time = s->last_heard + (f64)nat44_session_get_timeout(sm, s);
2440 if (ctx->now >= sess_timeout_time)
2442 if (is_fwd_bypass_session (s))
2445 ed_key.l_addr = s->out2in.addr;
2446 ed_key.r_addr = s->ext_host_addr;
2447 ed_key.fib_index = s->out2in.fib_index;
2448 if (snat_is_unk_proto_session (s))
2450 ed_key.proto = s->in2out.port;
2456 ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
2457 ed_key.l_port = s->out2in.port;
2458 ed_key.r_port = s->ext_host_port;
2460 ed_kv.key[0] = ed_key.as_u64[0];
2461 ed_kv.key[1] = ed_key.as_u64[1];
2462 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
2463 nat_log_warn ("out2in_ed key del failed");
2465 if (snat_is_unk_proto_session (s))
2468 snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
2469 s->out2in.addr.as_u32,
2473 s->in2out.fib_index);
2475 if (is_twice_nat_session (s))
2477 for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
2479 key.protocol = s->in2out.protocol;
2480 key.port = s->ext_host_nat_port;
2481 a = sm->twice_nat_addresses + i;
2482 if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
2484 snat_free_outside_address_and_port (sm->twice_nat_addresses,
2485 ctx->thread_index, &key);
2491 if (snat_is_session_static (s))
2494 if (s->outside_address_index != ~0)
2495 snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
2498 nat44_delete_session (sm, s, ctx->thread_index);
2506 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
2507 ip4_header_t * ip0, icmp46_header_t * icmp0,
2508 u32 sw_if_index0, u32 rx_fib_index0,
2509 vlib_node_runtime_t * node, u32 next0, f64 now,
2510 u32 thread_index, snat_session_t ** p_s0)
2512 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2513 next0, thread_index, p_s0, 0);
2514 snat_session_t * s0 = *p_s0;
2515 if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
2518 if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
2519 snat_icmp_hairpinning(sm, b0, ip0, icmp0, sm->endpoint_dependent);
2521 nat44_session_update_counters (s0, now,
2522 vlib_buffer_length_in_chain (sm->vlib_main, b0));
2528 slow_path_ed (snat_main_t *sm,
2531 clib_bihash_kv_16_8_t *kv,
2532 snat_session_t ** sessionp,
2533 vlib_node_runtime_t * node,
2540 snat_session_key_t key0, key1;
2541 lb_nat_type_t lb = 0, is_sm = 0;
2542 u32 address_index = ~0;
2543 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2544 nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
2545 u32 proto = ip_proto_to_snat_proto (key->proto);
2546 nat_outside_fib_t *outside_fib;
2547 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2548 fib_prefix_t pfx = {
2549 .fp_proto = FIB_PROTOCOL_IP4,
2552 .ip4.as_u32 = key->r_addr.as_u32,
2555 nat44_is_idle_session_ctx_t ctx;
2557 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
2559 b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2560 nat_ipfix_logging_max_sessions(sm->max_translations);
2561 nat_log_notice ("maximum sessions exceeded");
2562 return SNAT_IN2OUT_NEXT_DROP;
2565 key0.addr = key->l_addr;
2566 key0.port = key->l_port;
2567 key1.protocol = key0.protocol = proto;
2568 key0.fib_index = rx_fib_index;
2569 key1.fib_index = sm->outside_fib_index;
2570 /* First try to match static mapping by local address and port */
2571 if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb, 0))
2573 /* Try to create dynamic translation */
2574 if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
2575 thread_index, &key1,
2577 sm->port_per_thread,
2578 tsm->snat_thread_index))
2580 nat_log_notice ("addresses exhausted");
2581 b->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2582 return SNAT_IN2OUT_NEXT_DROP;
2588 u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
2591 nat_log_warn ("create NAT user failed");
2593 snat_free_outside_address_and_port (sm->addresses,
2594 thread_index, &key1);
2595 return SNAT_IN2OUT_NEXT_DROP;
2598 s = nat_ed_session_alloc (sm, u, thread_index);
2601 nat44_delete_user_with_no_session (sm, u, thread_index);
2602 nat_log_warn ("create NAT session failed");
2604 snat_free_outside_address_and_port (sm->addresses,
2605 thread_index, &key1);
2606 return SNAT_IN2OUT_NEXT_DROP;
2609 user_session_increment (sm, u, is_sm);
2611 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2613 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
2614 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2615 s->outside_address_index = address_index;
2616 s->ext_host_addr = key->r_addr;
2617 s->ext_host_port = key->r_port;
2620 s->out2in.protocol = key0.protocol;
2622 switch (vec_len (sm->outside_fibs))
2625 s->out2in.fib_index = sm->outside_fib_index;
2628 s->out2in.fib_index = sm->outside_fibs[0].fib_index;
2631 vec_foreach (outside_fib, sm->outside_fibs)
2633 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2634 if (FIB_NODE_INDEX_INVALID != fei)
2636 if (fib_entry_get_resolving_interface (fei) != ~0)
2638 s->out2in.fib_index = outside_fib->fib_index;
2646 /* Add to lookup tables */
2647 kv->value = s - tsm->sessions;
2649 ctx.thread_index = thread_index;
2650 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
2651 nat44_i2o_ed_is_idle_session_cb,
2653 nat_log_notice ("in2out-ed key add failed");
2655 make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
2656 key1.port, key->r_port);
2657 kv->value = s - tsm->sessions;
2658 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
2659 nat44_o2i_ed_is_idle_session_cb,
2661 nat_log_notice ("out2in-ed key add failed");
2666 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
2667 s->out2in.addr.as_u32,
2671 s->in2out.fib_index);
2675 static_always_inline int
2676 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
2677 u32 sw_if_index, ip4_header_t * ip, u32 proto,
2678 u32 rx_fib_index, u32 thread_index)
2680 udp_header_t *udp = ip4_next_header (ip);
2681 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2682 clib_bihash_kv_16_8_t kv, value;
2683 snat_session_key_t key0, key1;
2685 make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
2686 sm->outside_fib_index, udp->dst_port, udp->src_port);
2688 /* NAT packet aimed at external address if */
2689 /* has active sessions */
2690 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2692 key0.addr = ip->dst_address;
2693 key0.port = udp->dst_port;
2694 key0.protocol = proto;
2695 key0.fib_index = sm->outside_fib_index;
2696 /* or is static mappings */
2697 if (!snat_static_mapping_match(sm, key0, &key1, 1, 0, 0, 0, 0))
2703 if (sm->forwarding_enabled)
2706 return snat_not_translate_fast(sm, node, sw_if_index, ip, proto, rx_fib_index);
2709 static_always_inline int
2710 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
2711 u32 thread_index, f64 now,
2712 vlib_main_t * vm, vlib_buffer_t * b)
2714 nat_ed_ses_key_t key;
2715 clib_bihash_kv_16_8_t kv, value;
2717 snat_session_t *s = 0;
2718 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2720 if (!sm->forwarding_enabled)
2723 if (ip->protocol == IP_PROTOCOL_ICMP)
2725 key.as_u64[0] = key.as_u64[1] = 0;
2726 if (icmp_get_ed_key (ip, &key))
2729 kv.key[0] = key.as_u64[0];
2730 kv.key[1] = key.as_u64[1];
2732 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
2734 udp = ip4_next_header(ip);
2735 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
2736 udp->src_port, udp->dst_port);
2740 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
2744 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2746 s = pool_elt_at_index (tsm->sessions, value.value);
2747 if (is_fwd_bypass_session (s))
2749 if (ip->protocol == IP_PROTOCOL_TCP)
2751 tcp_header_t *tcp = ip4_next_header(ip);
2752 if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
2756 nat44_session_update_counters (s, now,
2757 vlib_buffer_length_in_chain (vm, b));
2767 static_always_inline int
2768 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
2769 u8 proto, u16 src_port, u16 dst_port,
2770 u32 thread_index, u32 rx_sw_if_index,
2773 clib_bihash_kv_16_8_t kv, value;
2774 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2775 snat_interface_t *i;
2777 u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
2778 u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
2781 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
2782 src_port, dst_port);
2783 if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2787 make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
2788 dst_port, src_port);
2789 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2791 s = pool_elt_at_index (tsm->sessions, value.value);
2792 if (is_fwd_bypass_session (s))
2796 pool_foreach (i, sm->output_feature_interfaces,
2798 if ((nat_interface_is_inside(i)) && (rx_sw_if_index == i->sw_if_index))
2808 icmp_match_in2out_ed(snat_main_t *sm, vlib_node_runtime_t *node,
2809 u32 thread_index, vlib_buffer_t *b, ip4_header_t *ip,
2810 u8 *p_proto, snat_session_key_t *p_value,
2811 u8 *p_dont_translate, void *d, void *e)
2813 icmp46_header_t *icmp;
2816 nat_ed_ses_key_t key;
2817 snat_session_t *s = 0;
2818 u8 dont_translate = 0;
2819 clib_bihash_kv_16_8_t kv, value;
2822 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2824 icmp = (icmp46_header_t *) ip4_next_header (ip);
2825 sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
2826 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2828 key.as_u64[0] = key.as_u64[1] = 0;
2829 err = icmp_get_ed_key (ip, &key);
2832 b->error = node->errors[err];
2833 next = SNAT_IN2OUT_NEXT_DROP;
2836 key.fib_index = rx_fib_index;
2838 kv.key[0] = key.as_u64[0];
2839 kv.key[1] = key.as_u64[1];
2841 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2843 if (vnet_buffer(b)->sw_if_index[VLIB_TX] != ~0)
2845 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(sm, ip,
2846 key.proto, key.l_port, key.r_port, thread_index, sw_if_index,
2847 vnet_buffer(b)->sw_if_index[VLIB_TX])))
2855 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node, sw_if_index,
2856 ip, SNAT_PROTOCOL_ICMP, rx_fib_index, thread_index)))
2863 if (PREDICT_FALSE(icmp_is_error_message (icmp)))
2865 b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2866 next = SNAT_IN2OUT_NEXT_DROP;
2870 next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
2871 thread_index, vlib_time_now (sm->vlib_main));
2873 if (PREDICT_FALSE (next == SNAT_IN2OUT_NEXT_DROP))
2878 if (PREDICT_FALSE(icmp->type != ICMP4_echo_request &&
2879 icmp->type != ICMP4_echo_reply &&
2880 !icmp_is_error_message (icmp)))
2882 b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2883 next = SNAT_IN2OUT_NEXT_DROP;
2887 s = pool_elt_at_index (tsm->sessions, value.value);
2890 *p_proto = ip_proto_to_snat_proto (key.proto);
2893 *p_value = s->out2in;
2894 *p_dont_translate = dont_translate;
2896 *(snat_session_t**)d = s;
2901 nat44_ed_hairpinning_unknown_proto (snat_main_t *sm,
2905 u32 old_addr, new_addr = 0, ti = 0;
2906 clib_bihash_kv_8_8_t kv, value;
2907 clib_bihash_kv_16_8_t s_kv, s_value;
2908 snat_static_mapping_t *m;
2911 snat_main_per_thread_data_t *tsm;
2913 if (sm->num_workers > 1)
2914 ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
2916 ti = sm->num_workers;
2917 tsm = &sm->per_thread_data[ti];
2919 old_addr = ip->dst_address.as_u32;
2920 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2921 sm->outside_fib_index, 0, 0);
2922 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2924 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
2925 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2928 m = pool_elt_at_index (sm->static_mappings, value.value);
2929 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2930 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
2931 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
2935 s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
2936 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2937 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2938 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
2941 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2942 ip->checksum = ip_csum_fold (sum);
2945 static snat_session_t *
2946 nat44_ed_in2out_unknown_proto (snat_main_t *sm,
2953 vlib_node_runtime_t * node)
2955 clib_bihash_kv_8_8_t kv, value;
2956 clib_bihash_kv_16_8_t s_kv, s_value;
2957 snat_static_mapping_t *m;
2958 u32 old_addr, new_addr = 0;
2961 dlist_elt_t *head, *elt;
2962 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2963 u32 elt_index, head_index, ses_index;
2965 u32 address_index = ~0, outside_fib_index = sm->outside_fib_index;
2968 nat_outside_fib_t *outside_fib;
2969 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2970 fib_prefix_t pfx = {
2971 .fp_proto = FIB_PROTOCOL_IP4,
2974 .ip4.as_u32 = ip->dst_address.as_u32,
2978 switch (vec_len (sm->outside_fibs))
2981 outside_fib_index = sm->outside_fib_index;
2984 outside_fib_index = sm->outside_fibs[0].fib_index;
2987 vec_foreach (outside_fib, sm->outside_fibs)
2989 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2990 if (FIB_NODE_INDEX_INVALID != fei)
2992 if (fib_entry_get_resolving_interface (fei) != ~0)
2994 outside_fib_index = outside_fib->fib_index;
3001 old_addr = ip->src_address.as_u32;
3003 make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
3004 rx_fib_index, 0, 0);
3006 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
3008 s = pool_elt_at_index (tsm->sessions, s_value.value);
3009 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
3013 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
3015 b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
3016 nat_ipfix_logging_max_sessions(sm->max_translations);
3017 nat_log_notice ("maximum sessions exceeded");
3021 u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
3025 nat_log_warn ("create NAT user failed");
3029 make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
3031 /* Try to find static mapping first */
3032 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
3034 m = pool_elt_at_index (sm->static_mappings, value.value);
3035 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
3039 /* Fallback to 3-tuple key */
3042 /* Choose same out address as for TCP/UDP session to same destination */
3043 head_index = u->sessions_per_user_list_head_index;
3044 head = pool_elt_at_index (tsm->list_pool, head_index);
3045 elt_index = head->next;
3046 if (PREDICT_FALSE (elt_index == ~0))
3050 elt = pool_elt_at_index (tsm->list_pool, elt_index);
3051 ses_index = elt->value;
3054 while (ses_index != ~0)
3056 s = pool_elt_at_index (tsm->sessions, ses_index);
3057 elt_index = elt->next;
3058 elt = pool_elt_at_index (tsm->list_pool, elt_index);
3059 ses_index = elt->value;
3061 if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
3063 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
3064 address_index = s->outside_address_index;
3066 make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
3067 ip->protocol, outside_fib_index, 0, 0);
3068 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
3075 for (i = 0; i < vec_len (sm->addresses); i++)
3077 make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
3078 ip->protocol, outside_fib_index, 0, 0);
3079 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
3081 new_addr = ip->src_address.as_u32 =
3082 sm->addresses[i].addr.as_u32;
3091 s = nat_ed_session_alloc (sm, u, thread_index);
3094 nat44_delete_user_with_no_session (sm, u, thread_index);
3095 nat_log_warn ("create NAT session failed");
3099 s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
3100 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
3101 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
3102 s->outside_address_index = address_index;
3103 s->out2in.addr.as_u32 = new_addr;
3104 s->out2in.fib_index = outside_fib_index;
3105 s->in2out.addr.as_u32 = old_addr;
3106 s->in2out.fib_index = rx_fib_index;
3107 s->in2out.port = s->out2in.port = ip->protocol;
3109 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
3110 user_session_increment (sm, u, is_sm);
3112 /* Add to lookup tables */
3113 make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
3114 rx_fib_index, 0, 0);
3115 s_kv.value = s - tsm->sessions;
3116 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
3117 nat_log_notice ("in2out key add failed");
3119 make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
3120 outside_fib_index, 0, 0);
3121 s_kv.value = s - tsm->sessions;
3122 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
3123 nat_log_notice ("out2in key add failed");
3126 /* Update IP checksum */
3128 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
3129 ip->checksum = ip_csum_fold (sum);
3132 nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
3135 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
3136 nat44_ed_hairpinning_unknown_proto(sm, b, ip);
3138 if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
3139 vnet_buffer(b)->sw_if_index[VLIB_TX] = outside_fib_index;
3145 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
3146 vlib_node_runtime_t * node,
3147 vlib_frame_t * frame, int is_slow_path,
3148 int is_output_feature)
3150 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
3151 snat_in2out_next_t next_index;
3152 snat_main_t *sm = &snat_main;
3153 f64 now = vlib_time_now (vm);
3154 u32 thread_index = vm->thread_index;
3155 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
3157 stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
3158 nat44_ed_in2out_node.index;
3160 from = vlib_frame_vector_args (frame);
3161 n_left_from = frame->n_vectors;
3162 next_index = node->cached_next_index;
3164 while (n_left_from > 0)
3168 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3170 while (n_left_from >= 4 && n_left_to_next >= 2)
3173 vlib_buffer_t *b0, *b1;
3174 u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3175 new_addr0, old_addr0;
3176 u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
3177 new_addr1, old_addr1;
3178 u16 old_port0, new_port0, old_port1, new_port1;
3179 ip4_header_t *ip0, *ip1;
3180 udp_header_t *udp0, *udp1;
3181 tcp_header_t *tcp0, *tcp1;
3182 icmp46_header_t *icmp0, *icmp1;
3183 snat_session_t *s0 = 0, *s1 = 0;
3184 clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
3185 ip_csum_t sum0, sum1;
3187 /* Prefetch next iteration. */
3189 vlib_buffer_t * p2, * p3;
3191 p2 = vlib_get_buffer (vm, from[2]);
3192 p3 = vlib_get_buffer (vm, from[3]);
3194 vlib_prefetch_buffer_header (p2, LOAD);
3195 vlib_prefetch_buffer_header (p3, LOAD);
3197 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3198 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3201 /* speculatively enqueue b0 and b1 to the current next frame */
3202 to_next[0] = bi0 = from[0];
3203 to_next[1] = bi1 = from[1];
3207 n_left_to_next -= 2;
3209 b0 = vlib_get_buffer (vm, bi0);
3210 b1 = vlib_get_buffer (vm, bi1);
3212 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3214 if (is_output_feature)
3215 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3217 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3220 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3221 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3224 if (PREDICT_FALSE(ip0->ttl == 1))
3226 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3227 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3228 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3230 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3234 udp0 = ip4_next_header (ip0);
3235 tcp0 = (tcp_header_t *) udp0;
3236 icmp0 = (icmp46_header_t *) udp0;
3237 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3241 if (PREDICT_FALSE (proto0 == ~0))
3243 s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3245 thread_index, now, vm,
3248 next0 = SNAT_IN2OUT_NEXT_DROP;
3252 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3254 next0 = icmp_in2out_ed_slow_path
3255 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3256 next0, now, thread_index, &s0);
3262 if (is_output_feature)
3264 if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3265 sm, ip0, thread_index, now, vm, b0)))
3269 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3271 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3275 if (ip4_is_fragment (ip0))
3277 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3278 next0 = SNAT_IN2OUT_NEXT_DROP;
3283 make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3284 rx_fib_index0, udp0->src_port, udp0->dst_port);
3286 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3290 if (is_output_feature)
3292 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3293 sm, ip0, ip0->protocol, udp0->src_port,
3294 udp0->dst_port, thread_index, sw_if_index0,
3295 vnet_buffer(b0)->sw_if_index[VLIB_TX])))
3300 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3301 sw_if_index0, ip0, proto0, rx_fib_index0,
3306 next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3307 next0, thread_index, now);
3309 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3314 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3320 s0 = pool_elt_at_index (tsm->sessions, value0.value);
3323 b0->flags |= VNET_BUFFER_F_IS_NATED;
3325 if (!is_output_feature)
3326 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3328 old_addr0 = ip0->src_address.as_u32;
3329 new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3330 sum0 = ip0->checksum;
3331 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3333 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3334 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3335 s0->ext_host_addr.as_u32, ip4_header_t,
3337 ip0->checksum = ip_csum_fold (sum0);
3339 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3341 old_port0 = tcp0->src_port;
3342 new_port0 = tcp0->src_port = s0->out2in.port;
3344 sum0 = tcp0->checksum;
3345 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3347 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3349 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3351 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3352 s0->ext_host_addr.as_u32,
3353 ip4_header_t, dst_address);
3354 sum0 = ip_csum_update (sum0, tcp0->dst_port,
3355 s0->ext_host_port, ip4_header_t,
3357 tcp0->dst_port = s0->ext_host_port;
3358 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3360 mss_clamping (sm, tcp0, &sum0);
3361 tcp0->checksum = ip_csum_fold(sum0);
3362 if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3367 udp0->src_port = s0->out2in.port;
3369 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3371 udp0->dst_port = s0->ext_host_port;
3372 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3377 nat44_session_update_counters (s0, now,
3378 vlib_buffer_length_in_chain (vm, b0));
3381 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3382 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3384 snat_in2out_trace_t *t =
3385 vlib_add_trace (vm, node, b0, sizeof (*t));
3386 t->is_slow_path = is_slow_path;
3387 t->sw_if_index = sw_if_index0;
3388 t->next_index = next0;
3389 t->session_index = ~0;
3391 t->session_index = s0 - tsm->sessions;
3394 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3397 next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3399 if (is_output_feature)
3400 iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
3402 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
3405 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3406 rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3409 if (PREDICT_FALSE(ip1->ttl == 1))
3411 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3412 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3413 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3415 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3419 udp1 = ip4_next_header (ip1);
3420 tcp1 = (tcp_header_t *) udp1;
3421 icmp1 = (icmp46_header_t *) udp1;
3422 proto1 = ip_proto_to_snat_proto (ip1->protocol);
3426 if (PREDICT_FALSE (proto1 == ~0))
3428 s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
3430 thread_index, now, vm,
3433 next1 = SNAT_IN2OUT_NEXT_DROP;
3437 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
3439 next1 = icmp_in2out_ed_slow_path
3440 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
3441 next1, now, thread_index, &s1);
3447 if (is_output_feature)
3449 if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3450 sm, ip1, thread_index, now, vm, b1)))
3454 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
3456 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3460 if (ip4_is_fragment (ip1))
3462 b1->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3463 next1 = SNAT_IN2OUT_NEXT_DROP;
3468 make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address, ip1->protocol,
3469 rx_fib_index1, udp1->src_port, udp1->dst_port);
3471 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
3475 if (is_output_feature)
3477 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3478 sm, ip1, ip1->protocol, udp1->src_port,
3479 udp1->dst_port, thread_index, sw_if_index1,
3480 vnet_buffer(b1)->sw_if_index[VLIB_TX])))
3485 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3486 sw_if_index1, ip1, proto1, rx_fib_index1,
3491 next1 = slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
3492 next1, thread_index, now);
3494 if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
3499 next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3505 s1 = pool_elt_at_index (tsm->sessions, value1.value);
3508 b1->flags |= VNET_BUFFER_F_IS_NATED;
3510 if (!is_output_feature)
3511 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
3513 old_addr1 = ip1->src_address.as_u32;
3514 new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
3515 sum1 = ip1->checksum;
3516 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3518 if (PREDICT_FALSE (is_twice_nat_session (s1)))
3519 sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3520 s1->ext_host_addr.as_u32, ip4_header_t,
3522 ip1->checksum = ip_csum_fold (sum1);
3524 if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
3526 old_port1 = tcp1->src_port;
3527 new_port1 = tcp1->src_port = s1->out2in.port;
3529 sum1 = tcp1->checksum;
3530 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3532 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
3534 if (PREDICT_FALSE (is_twice_nat_session (s1)))
3536 sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3537 s1->ext_host_addr.as_u32,
3538 ip4_header_t, dst_address);
3539 sum1 = ip_csum_update (sum1, tcp1->dst_port,
3540 s1->ext_host_port, ip4_header_t,
3542 tcp1->dst_port = s1->ext_host_port;
3543 ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3545 tcp1->checksum = ip_csum_fold(sum1);
3546 mss_clamping (sm, tcp1, &sum1);
3547 if (nat44_set_tcp_session_state_i2o (sm, s1, tcp1, thread_index))
3552 udp1->src_port = s1->out2in.port;
3554 if (PREDICT_FALSE (is_twice_nat_session (s1)))
3556 udp1->dst_port = s1->ext_host_port;
3557 ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3562 nat44_session_update_counters (s1, now,
3563 vlib_buffer_length_in_chain (vm, b1));
3566 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3567 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3569 snat_in2out_trace_t *t =
3570 vlib_add_trace (vm, node, b1, sizeof (*t));
3571 t->is_slow_path = is_slow_path;
3572 t->sw_if_index = sw_if_index1;
3573 t->next_index = next1;
3574 t->session_index = ~0;
3576 t->session_index = s1 - tsm->sessions;
3579 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3581 /* verify speculative enqueues, maybe switch current next frame */
3582 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3583 to_next, n_left_to_next,
3584 bi0, bi1, next0, next1);
3587 while (n_left_from > 0 && n_left_to_next > 0)
3591 u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3592 new_addr0, old_addr0;
3593 u16 old_port0, new_port0;
3597 icmp46_header_t * icmp0;
3598 snat_session_t *s0 = 0;
3599 clib_bihash_kv_16_8_t kv0, value0;
3602 /* speculatively enqueue b0 to the current next frame */
3608 n_left_to_next -= 1;
3610 b0 = vlib_get_buffer (vm, bi0);
3611 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3613 if (is_output_feature)
3614 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3616 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3619 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3620 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3623 if (PREDICT_FALSE(ip0->ttl == 1))
3625 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3626 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3627 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3629 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3633 udp0 = ip4_next_header (ip0);
3634 tcp0 = (tcp_header_t *) udp0;
3635 icmp0 = (icmp46_header_t *) udp0;
3636 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3640 if (PREDICT_FALSE (proto0 == ~0))
3642 s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3644 thread_index, now, vm,
3647 next0 = SNAT_IN2OUT_NEXT_DROP;
3651 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3653 next0 = icmp_in2out_ed_slow_path
3654 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3655 next0, now, thread_index, &s0);
3661 if (is_output_feature)
3663 if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3664 sm, ip0, thread_index, now, vm, b0)))
3668 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3670 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3674 if (ip4_is_fragment (ip0))
3676 b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3677 next0 = SNAT_IN2OUT_NEXT_DROP;
3682 make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3683 rx_fib_index0, udp0->src_port, udp0->dst_port);
3685 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3689 if (is_output_feature)
3691 if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3692 sm, ip0, ip0->protocol, udp0->src_port,
3693 udp0->dst_port, thread_index, sw_if_index0,
3694 vnet_buffer(b0)->sw_if_index[VLIB_TX])))
3699 if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3700 sw_if_index0, ip0, proto0, rx_fib_index0,
3705 next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3706 next0, thread_index, now);
3708 if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3713 next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3719 s0 = pool_elt_at_index (tsm->sessions, value0.value);
3722 b0->flags |= VNET_BUFFER_F_IS_NATED;
3724 if (!is_output_feature)
3725 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3727 old_addr0 = ip0->src_address.as_u32;
3728 new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3729 sum0 = ip0->checksum;
3730 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3732 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3733 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3734 s0->ext_host_addr.as_u32, ip4_header_t,
3736 ip0->checksum = ip_csum_fold (sum0);
3738 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3740 old_port0 = tcp0->src_port;
3741 new_port0 = tcp0->src_port = s0->out2in.port;
3743 sum0 = tcp0->checksum;
3744 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3746 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3748 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3750 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3751 s0->ext_host_addr.as_u32,
3752 ip4_header_t, dst_address);
3753 sum0 = ip_csum_update (sum0, tcp0->dst_port,
3754 s0->ext_host_port, ip4_header_t,
3756 tcp0->dst_port = s0->ext_host_port;
3757 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3759 mss_clamping (sm, tcp0, &sum0);
3760 tcp0->checksum = ip_csum_fold(sum0);
3761 if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3766 udp0->src_port = s0->out2in.port;
3768 if (PREDICT_FALSE (is_twice_nat_session (s0)))
3770 udp0->dst_port = s0->ext_host_port;
3771 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3776 nat44_session_update_counters (s0, now,
3777 vlib_buffer_length_in_chain (vm, b0));
3780 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3781 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3783 snat_in2out_trace_t *t =
3784 vlib_add_trace (vm, node, b0, sizeof (*t));
3785 t->is_slow_path = is_slow_path;
3786 t->sw_if_index = sw_if_index0;
3787 t->next_index = next0;
3788 t->session_index = ~0;
3790 t->session_index = s0 - tsm->sessions;
3793 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3795 /* verify speculative enqueue, maybe switch current next frame */
3796 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3797 to_next, n_left_to_next,
3801 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3804 vlib_node_increment_counter (vm, stats_node_index,
3805 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3807 return frame->n_vectors;
3811 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
3812 vlib_node_runtime_t * node,
3813 vlib_frame_t * frame)
3815 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
3818 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
3819 .function = nat44_ed_in2out_fast_path_fn,
3820 .name = "nat44-ed-in2out",
3821 .vector_size = sizeof (u32),
3822 .format_trace = format_snat_in2out_trace,
3823 .type = VLIB_NODE_TYPE_INTERNAL,
3825 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3826 .error_strings = snat_in2out_error_strings,
3828 .runtime_data_bytes = sizeof (snat_runtime_t),
3830 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3832 /* edit / add dispositions here */
3834 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3835 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3836 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3837 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3838 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3842 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node, nat44_ed_in2out_fast_path_fn);
3845 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
3846 vlib_node_runtime_t * node,
3847 vlib_frame_t * frame)
3849 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
3852 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
3853 .function = nat44_ed_in2out_output_fast_path_fn,
3854 .name = "nat44-ed-in2out-output",
3855 .vector_size = sizeof (u32),
3856 .format_trace = format_snat_in2out_trace,
3857 .type = VLIB_NODE_TYPE_INTERNAL,
3859 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3860 .error_strings = snat_in2out_error_strings,
3862 .runtime_data_bytes = sizeof (snat_runtime_t),
3864 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3866 /* edit / add dispositions here */
3868 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3869 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3870 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3871 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3872 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3876 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
3877 nat44_ed_in2out_output_fast_path_fn);
3880 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
3881 vlib_node_runtime_t * node,
3882 vlib_frame_t * frame)
3884 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
3887 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
3888 .function = nat44_ed_in2out_slow_path_fn,
3889 .name = "nat44-ed-in2out-slowpath",
3890 .vector_size = sizeof (u32),
3891 .format_trace = format_snat_in2out_trace,
3892 .type = VLIB_NODE_TYPE_INTERNAL,
3894 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3895 .error_strings = snat_in2out_error_strings,
3897 .runtime_data_bytes = sizeof (snat_runtime_t),
3899 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3901 /* edit / add dispositions here */
3903 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3904 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3905 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3906 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3907 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3911 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
3912 nat44_ed_in2out_slow_path_fn);
3915 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
3916 vlib_node_runtime_t * node,
3917 vlib_frame_t * frame)
3919 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
3922 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
3923 .function = nat44_ed_in2out_output_slow_path_fn,
3924 .name = "nat44-ed-in2out-output-slowpath",
3925 .vector_size = sizeof (u32),
3926 .format_trace = format_snat_in2out_trace,
3927 .type = VLIB_NODE_TYPE_INTERNAL,
3929 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3930 .error_strings = snat_in2out_error_strings,
3932 .runtime_data_bytes = sizeof (snat_runtime_t),
3934 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3936 /* edit / add dispositions here */
3938 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3939 [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3940 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3941 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3942 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3946 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
3947 nat44_ed_in2out_output_slow_path_fn);
3949 /**************************/
3950 /*** deterministic mode ***/
3951 /**************************/
3953 snat_det_in2out_node_fn (vlib_main_t * vm,
3954 vlib_node_runtime_t * node,
3955 vlib_frame_t * frame)
3957 u32 n_left_from, * from, * to_next;
3958 snat_in2out_next_t next_index;
3959 u32 pkts_processed = 0;
3960 snat_main_t * sm = &snat_main;
3961 u32 now = (u32) vlib_time_now (vm);
3962 u32 thread_index = vm->thread_index;
3964 from = vlib_frame_vector_args (frame);
3965 n_left_from = frame->n_vectors;
3966 next_index = node->cached_next_index;
3968 while (n_left_from > 0)
3972 vlib_get_next_frame (vm, node, next_index,
3973 to_next, n_left_to_next);
3975 while (n_left_from >= 4 && n_left_to_next >= 2)
3978 vlib_buffer_t * b0, * b1;
3980 u32 sw_if_index0, sw_if_index1;
3981 ip4_header_t * ip0, * ip1;
3982 ip_csum_t sum0, sum1;
3983 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
3984 u16 old_port0, new_port0, lo_port0, i0;
3985 u16 old_port1, new_port1, lo_port1, i1;
3986 udp_header_t * udp0, * udp1;
3987 tcp_header_t * tcp0, * tcp1;
3989 snat_det_out_key_t key0, key1;
3990 snat_det_map_t * dm0, * dm1;
3991 snat_det_session_t * ses0 = 0, * ses1 = 0;
3992 u32 rx_fib_index0, rx_fib_index1;
3993 icmp46_header_t * icmp0, * icmp1;
3995 /* Prefetch next iteration. */
3997 vlib_buffer_t * p2, * p3;
3999 p2 = vlib_get_buffer (vm, from[2]);
4000 p3 = vlib_get_buffer (vm, from[3]);
4002 vlib_prefetch_buffer_header (p2, LOAD);
4003 vlib_prefetch_buffer_header (p3, LOAD);
4005 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
4006 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
4009 /* speculatively enqueue b0 and b1 to the current next frame */
4010 to_next[0] = bi0 = from[0];
4011 to_next[1] = bi1 = from[1];
4015 n_left_to_next -= 2;
4017 b0 = vlib_get_buffer (vm, bi0);
4018 b1 = vlib_get_buffer (vm, bi1);
4020 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4021 next1 = SNAT_IN2OUT_NEXT_LOOKUP;
4023 ip0 = vlib_buffer_get_current (b0);
4024 udp0 = ip4_next_header (ip0);
4025 tcp0 = (tcp_header_t *) udp0;
4027 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4029 if (PREDICT_FALSE(ip0->ttl == 1))
4031 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4032 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4033 ICMP4_time_exceeded_ttl_exceeded_in_transit,
4035 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4039 proto0 = ip_proto_to_snat_proto (ip0->protocol);
4041 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4043 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4044 icmp0 = (icmp46_header_t *) udp0;
4046 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4047 rx_fib_index0, node, next0, thread_index,
4052 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4053 if (PREDICT_FALSE(!dm0))
4055 nat_log_info ("no match for internal host %U",
4056 format_ip4_address, &ip0->src_address);
4057 next0 = SNAT_IN2OUT_NEXT_DROP;
4058 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4062 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4064 key0.ext_host_addr = ip0->dst_address;
4065 key0.ext_host_port = tcp0->dst;
4067 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4068 if (PREDICT_FALSE(!ses0))
4070 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4072 key0.out_port = clib_host_to_net_u16 (lo_port0 +
4073 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4075 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4078 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4081 if (PREDICT_FALSE(!ses0))
4083 /* too many sessions for user, send ICMP error packet */
4085 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4086 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4087 ICMP4_destination_unreachable_destination_unreachable_host,
4089 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4094 new_port0 = ses0->out.out_port;
4096 old_addr0.as_u32 = ip0->src_address.as_u32;
4097 ip0->src_address.as_u32 = new_addr0.as_u32;
4098 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4100 sum0 = ip0->checksum;
4101 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4103 src_address /* changed member */);
4104 ip0->checksum = ip_csum_fold (sum0);
4106 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4108 if (tcp0->flags & TCP_FLAG_SYN)
4109 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4110 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4111 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4112 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4113 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4114 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4115 snat_det_ses_close(dm0, ses0);
4116 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4117 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4118 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4119 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4121 old_port0 = tcp0->src;
4122 tcp0->src = new_port0;
4124 sum0 = tcp0->checksum;
4125 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4127 dst_address /* changed member */);
4128 sum0 = ip_csum_update (sum0, old_port0, new_port0,
4129 ip4_header_t /* cheat */,
4130 length /* changed member */);
4131 mss_clamping (sm, tcp0, &sum0);
4132 tcp0->checksum = ip_csum_fold(sum0);
4136 ses0->state = SNAT_SESSION_UDP_ACTIVE;
4137 old_port0 = udp0->src_port;
4138 udp0->src_port = new_port0;
4144 case SNAT_SESSION_UDP_ACTIVE:
4145 ses0->expire = now + sm->udp_timeout;
4147 case SNAT_SESSION_TCP_SYN_SENT:
4148 case SNAT_SESSION_TCP_FIN_WAIT:
4149 case SNAT_SESSION_TCP_CLOSE_WAIT:
4150 case SNAT_SESSION_TCP_LAST_ACK:
4151 ses0->expire = now + sm->tcp_transitory_timeout;
4153 case SNAT_SESSION_TCP_ESTABLISHED:
4154 ses0->expire = now + sm->tcp_established_timeout;
4159 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4160 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4162 snat_in2out_trace_t *t =
4163 vlib_add_trace (vm, node, b0, sizeof (*t));
4164 t->is_slow_path = 0;
4165 t->sw_if_index = sw_if_index0;
4166 t->next_index = next0;
4167 t->session_index = ~0;
4169 t->session_index = ses0 - dm0->sessions;
4172 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4174 ip1 = vlib_buffer_get_current (b1);
4175 udp1 = ip4_next_header (ip1);
4176 tcp1 = (tcp_header_t *) udp1;
4178 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
4180 if (PREDICT_FALSE(ip1->ttl == 1))
4182 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4183 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
4184 ICMP4_time_exceeded_ttl_exceeded_in_transit,
4186 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4190 proto1 = ip_proto_to_snat_proto (ip1->protocol);
4192 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
4194 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
4195 icmp1 = (icmp46_header_t *) udp1;
4197 next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
4198 rx_fib_index1, node, next1, thread_index,
4203 dm1 = snat_det_map_by_user(sm, &ip1->src_address);
4204 if (PREDICT_FALSE(!dm1))
4206 nat_log_info ("no match for internal host %U",
4207 format_ip4_address, &ip0->src_address);
4208 next1 = SNAT_IN2OUT_NEXT_DROP;
4209 b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4213 snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
4215 key1.ext_host_addr = ip1->dst_address;
4216 key1.ext_host_port = tcp1->dst;
4218 ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
4219 if (PREDICT_FALSE(!ses1))
4221 for (i1 = 0; i1 < dm1->ports_per_host; i1++)
4223 key1.out_port = clib_host_to_net_u16 (lo_port1 +
4224 ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
4226 if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
4229 ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
4232 if (PREDICT_FALSE(!ses1))
4234 /* too many sessions for user, send ICMP error packet */
4236 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4237 icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
4238 ICMP4_destination_unreachable_destination_unreachable_host,
4240 next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4245 new_port1 = ses1->out.out_port;
4247 old_addr1.as_u32 = ip1->src_address.as_u32;
4248 ip1->src_address.as_u32 = new_addr1.as_u32;
4249 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4251 sum1 = ip1->checksum;
4252 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4254 src_address /* changed member */);
4255 ip1->checksum = ip_csum_fold (sum1);
4257 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
4259 if (tcp1->flags & TCP_FLAG_SYN)
4260 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
4261 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
4262 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4263 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
4264 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
4265 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
4266 snat_det_ses_close(dm1, ses1);
4267 else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4268 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
4269 else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
4270 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4272 old_port1 = tcp1->src;
4273 tcp1->src = new_port1;
4275 sum1 = tcp1->checksum;
4276 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4278 dst_address /* changed member */);
4279 sum1 = ip_csum_update (sum1, old_port1, new_port1,
4280 ip4_header_t /* cheat */,
4281 length /* changed member */);
4282 mss_clamping (sm, tcp1, &sum1);
4283 tcp1->checksum = ip_csum_fold(sum1);
4287 ses1->state = SNAT_SESSION_UDP_ACTIVE;
4288 old_port1 = udp1->src_port;
4289 udp1->src_port = new_port1;
4295 case SNAT_SESSION_UDP_ACTIVE:
4296 ses1->expire = now + sm->udp_timeout;
4298 case SNAT_SESSION_TCP_SYN_SENT:
4299 case SNAT_SESSION_TCP_FIN_WAIT:
4300 case SNAT_SESSION_TCP_CLOSE_WAIT:
4301 case SNAT_SESSION_TCP_LAST_ACK:
4302 ses1->expire = now + sm->tcp_transitory_timeout;
4304 case SNAT_SESSION_TCP_ESTABLISHED:
4305 ses1->expire = now + sm->tcp_established_timeout;
4310 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4311 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
4313 snat_in2out_trace_t *t =
4314 vlib_add_trace (vm, node, b1, sizeof (*t));
4315 t->is_slow_path = 0;
4316 t->sw_if_index = sw_if_index1;
4317 t->next_index = next1;
4318 t->session_index = ~0;
4320 t->session_index = ses1 - dm1->sessions;
4323 pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
4325 /* verify speculative enqueues, maybe switch current next frame */
4326 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
4327 to_next, n_left_to_next,
4328 bi0, bi1, next0, next1);
4331 while (n_left_from > 0 && n_left_to_next > 0)
4339 ip4_address_t new_addr0, old_addr0;
4340 u16 old_port0, new_port0, lo_port0, i0;
4341 udp_header_t * udp0;
4342 tcp_header_t * tcp0;
4344 snat_det_out_key_t key0;
4345 snat_det_map_t * dm0;
4346 snat_det_session_t * ses0 = 0;
4348 icmp46_header_t * icmp0;
4350 /* speculatively enqueue b0 to the current next frame */
4356 n_left_to_next -= 1;
4358 b0 = vlib_get_buffer (vm, bi0);
4359 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4361 ip0 = vlib_buffer_get_current (b0);
4362 udp0 = ip4_next_header (ip0);
4363 tcp0 = (tcp_header_t *) udp0;
4365 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4367 if (PREDICT_FALSE(ip0->ttl == 1))
4369 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4370 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4371 ICMP4_time_exceeded_ttl_exceeded_in_transit,
4373 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4377 proto0 = ip_proto_to_snat_proto (ip0->protocol);
4379 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4381 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4382 icmp0 = (icmp46_header_t *) udp0;
4384 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4385 rx_fib_index0, node, next0, thread_index,
4390 dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4391 if (PREDICT_FALSE(!dm0))
4393 nat_log_info ("no match for internal host %U",
4394 format_ip4_address, &ip0->src_address);
4395 next0 = SNAT_IN2OUT_NEXT_DROP;
4396 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4400 snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4402 key0.ext_host_addr = ip0->dst_address;
4403 key0.ext_host_port = tcp0->dst;
4405 ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4406 if (PREDICT_FALSE(!ses0))
4408 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4410 key0.out_port = clib_host_to_net_u16 (lo_port0 +
4411 ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4413 if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4416 ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4419 if (PREDICT_FALSE(!ses0))
4421 /* too many sessions for user, send ICMP error packet */
4423 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4424 icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4425 ICMP4_destination_unreachable_destination_unreachable_host,
4427 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4432 new_port0 = ses0->out.out_port;
4434 old_addr0.as_u32 = ip0->src_address.as_u32;
4435 ip0->src_address.as_u32 = new_addr0.as_u32;
4436 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4438 sum0 = ip0->checksum;
4439 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4441 src_address /* changed member */);
4442 ip0->checksum = ip_csum_fold (sum0);
4444 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4446 if (tcp0->flags & TCP_FLAG_SYN)
4447 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4448 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4449 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4450 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4451 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4452 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4453 snat_det_ses_close(dm0, ses0);
4454 else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4455 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4456 else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4457 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4459 old_port0 = tcp0->src;
4460 tcp0->src = new_port0;
4462 sum0 = tcp0->checksum;
4463 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4465 dst_address /* changed member */);
4466 sum0 = ip_csum_update (sum0, old_port0, new_port0,
4467 ip4_header_t /* cheat */,
4468 length /* changed member */);
4469 mss_clamping (sm, tcp0, &sum0);
4470 tcp0->checksum = ip_csum_fold(sum0);
4474 ses0->state = SNAT_SESSION_UDP_ACTIVE;
4475 old_port0 = udp0->src_port;
4476 udp0->src_port = new_port0;
4482 case SNAT_SESSION_UDP_ACTIVE:
4483 ses0->expire = now + sm->udp_timeout;
4485 case SNAT_SESSION_TCP_SYN_SENT:
4486 case SNAT_SESSION_TCP_FIN_WAIT:
4487 case SNAT_SESSION_TCP_CLOSE_WAIT:
4488 case SNAT_SESSION_TCP_LAST_ACK:
4489 ses0->expire = now + sm->tcp_transitory_timeout;
4491 case SNAT_SESSION_TCP_ESTABLISHED:
4492 ses0->expire = now + sm->tcp_established_timeout;
4497 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4498 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4500 snat_in2out_trace_t *t =
4501 vlib_add_trace (vm, node, b0, sizeof (*t));
4502 t->is_slow_path = 0;
4503 t->sw_if_index = sw_if_index0;
4504 t->next_index = next0;
4505 t->session_index = ~0;
4507 t->session_index = ses0 - dm0->sessions;
4510 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4512 /* verify speculative enqueue, maybe switch current next frame */
4513 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4514 to_next, n_left_to_next,
4518 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4521 vlib_node_increment_counter (vm, snat_det_in2out_node.index,
4522 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4524 return frame->n_vectors;
4527 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
4528 .function = snat_det_in2out_node_fn,
4529 .name = "nat44-det-in2out",
4530 .vector_size = sizeof (u32),
4531 .format_trace = format_snat_in2out_trace,
4532 .type = VLIB_NODE_TYPE_INTERNAL,
4534 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4535 .error_strings = snat_in2out_error_strings,
4537 .runtime_data_bytes = sizeof (snat_runtime_t),
4541 /* edit / add dispositions here */
4543 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4544 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4545 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4549 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
4552 * Get address and port values to be used for ICMP packet translation
4553 * and create session if needed
4555 * @param[in,out] sm NAT main
4556 * @param[in,out] node NAT node runtime
4557 * @param[in] thread_index thread index
4558 * @param[in,out] b0 buffer containing packet to be translated
4559 * @param[out] p_proto protocol used for matching
4560 * @param[out] p_value address and port after NAT translation
4561 * @param[out] p_dont_translate if packet should not be translated
4562 * @param d optional parameter
4563 * @param e optional parameter
4565 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
4566 u32 thread_index, vlib_buffer_t *b0,
4567 ip4_header_t *ip0, u8 *p_proto,
4568 snat_session_key_t *p_value,
4569 u8 *p_dont_translate, void *d, void *e)
4571 icmp46_header_t *icmp0;
4575 snat_det_out_key_t key0;
4576 u8 dont_translate = 0;
4578 icmp_echo_header_t *echo0, *inner_echo0 = 0;
4579 ip4_header_t *inner_ip0;
4580 void *l4_header = 0;
4581 icmp46_header_t *inner_icmp0;
4582 snat_det_map_t * dm0 = 0;
4583 ip4_address_t new_addr0;
4585 snat_det_session_t * ses0 = 0;
4586 ip4_address_t in_addr;
4589 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
4590 echo0 = (icmp_echo_header_t *)(icmp0+1);
4591 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4592 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
4594 if (!icmp_is_error_message (icmp0))
4596 protocol = SNAT_PROTOCOL_ICMP;
4597 in_addr = ip0->src_address;
4598 in_port = echo0->identifier;
4602 inner_ip0 = (ip4_header_t *)(echo0+1);
4603 l4_header = ip4_next_header (inner_ip0);
4604 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
4605 in_addr = inner_ip0->dst_address;
4608 case SNAT_PROTOCOL_ICMP:
4609 inner_icmp0 = (icmp46_header_t*)l4_header;
4610 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
4611 in_port = inner_echo0->identifier;
4613 case SNAT_PROTOCOL_UDP:
4614 case SNAT_PROTOCOL_TCP:
4615 in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
4618 b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
4619 next0 = SNAT_IN2OUT_NEXT_DROP;
4624 dm0 = snat_det_map_by_user(sm, &in_addr);
4625 if (PREDICT_FALSE(!dm0))
4627 nat_log_info ("no match for internal host %U",
4628 format_ip4_address, &in_addr);
4629 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4630 IP_PROTOCOL_ICMP, rx_fib_index0)))
4635 next0 = SNAT_IN2OUT_NEXT_DROP;
4636 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4640 snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
4642 key0.ext_host_addr = ip0->dst_address;
4643 key0.ext_host_port = 0;
4645 ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
4646 if (PREDICT_FALSE(!ses0))
4648 if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4649 IP_PROTOCOL_ICMP, rx_fib_index0)))
4654 if (icmp0->type != ICMP4_echo_request)
4656 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4657 next0 = SNAT_IN2OUT_NEXT_DROP;
4660 for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4662 key0.out_port = clib_host_to_net_u16 (lo_port0 +
4663 ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
4665 if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
4668 ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
4671 if (PREDICT_FALSE(!ses0))
4673 next0 = SNAT_IN2OUT_NEXT_DROP;
4674 b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
4679 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
4680 !icmp_is_error_message (icmp0)))
4682 b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4683 next0 = SNAT_IN2OUT_NEXT_DROP;
4687 u32 now = (u32) vlib_time_now (sm->vlib_main);
4689 ses0->state = SNAT_SESSION_ICMP_ACTIVE;
4690 ses0->expire = now + sm->icmp_timeout;
4693 *p_proto = protocol;
4696 p_value->addr = new_addr0;
4697 p_value->fib_index = sm->outside_fib_index;
4698 p_value->port = ses0->out.out_port;
4700 *p_dont_translate = dont_translate;
4702 *(snat_det_session_t**)d = ses0;
4704 *(snat_det_map_t**)e = dm0;
4708 /**********************/
4709 /*** worker handoff ***/
4710 /**********************/
4712 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
4713 vlib_node_runtime_t * node,
4714 vlib_frame_t * frame,
4717 snat_main_t *sm = &snat_main;
4718 vlib_thread_main_t *tm = vlib_get_thread_main ();
4719 u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
4720 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
4721 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
4723 vlib_frame_queue_elt_t *hf = 0;
4724 vlib_frame_queue_t *fq;
4725 vlib_frame_t *f = 0;
4727 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
4728 u32 next_worker_index = 0;
4729 u32 current_worker_index = ~0;
4730 u32 thread_index = vm->thread_index;
4733 vlib_frame_t *d = 0;
4735 ASSERT (vec_len (sm->workers));
4739 fq_index = sm->fq_in2out_output_index;
4740 to_node_index = sm->in2out_output_node_index;
4744 fq_index = sm->fq_in2out_index;
4745 to_node_index = sm->in2out_node_index;
4748 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
4750 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
4752 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
4753 tm->n_vlib_mains - 1,
4754 (vlib_frame_queue_t *) (~0));
4757 from = vlib_frame_vector_args (frame);
4758 n_left_from = frame->n_vectors;
4760 while (n_left_from > 0)
4773 b0 = vlib_get_buffer (vm, bi0);
4775 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
4776 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4778 ip0 = vlib_buffer_get_current (b0);
4780 next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
4782 if (PREDICT_FALSE (next_worker_index != thread_index))
4786 if (next_worker_index != current_worker_index)
4788 fq = is_vlib_frame_queue_congested (
4789 fq_index, next_worker_index, NAT_FQ_NELTS - 2,
4790 congested_handoff_queue_by_worker_index);
4794 /* if this is 1st frame */
4797 d = vlib_get_frame_to_node (vm, sm->error_node_index);
4798 to_next_drop = vlib_frame_vector_args (d);
4801 to_next_drop[0] = bi0;
4804 b0->error = node->errors[SNAT_IN2OUT_ERROR_FQ_CONGESTED];
4809 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4811 hf = vlib_get_worker_handoff_queue_elt (fq_index,
4813 handoff_queue_elt_by_worker_index);
4815 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
4816 to_next_worker = &hf->buffer_index[hf->n_vectors];
4817 current_worker_index = next_worker_index;
4820 /* enqueue to correct worker thread */
4821 to_next_worker[0] = bi0;
4823 n_left_to_next_worker--;
4825 if (n_left_to_next_worker == 0)
4827 hf->n_vectors = VLIB_FRAME_SIZE;
4828 vlib_put_frame_queue_elt (hf);
4829 current_worker_index = ~0;
4830 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
4837 /* if this is 1st frame */
4840 f = vlib_get_frame_to_node (vm, to_node_index);
4841 to_next = vlib_frame_vector_args (f);
4850 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
4851 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4853 snat_in2out_worker_handoff_trace_t *t =
4854 vlib_add_trace (vm, node, b0, sizeof (*t));
4855 t->next_worker_index = next_worker_index;
4856 t->do_handoff = do_handoff;
4861 vlib_put_frame_to_node (vm, to_node_index, f);
4864 vlib_put_frame_to_node (vm, sm->error_node_index, d);
4867 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4869 /* Ship frames to the worker nodes */
4870 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
4872 if (handoff_queue_elt_by_worker_index[i])
4874 hf = handoff_queue_elt_by_worker_index[i];
4876 * It works better to let the handoff node
4877 * rate-adapt, always ship the handoff queue element.
4879 if (1 || hf->n_vectors == hf->last_n_vectors)
4881 vlib_put_frame_queue_elt (hf);
4882 handoff_queue_elt_by_worker_index[i] = 0;
4885 hf->last_n_vectors = hf->n_vectors;
4887 congested_handoff_queue_by_worker_index[i] =
4888 (vlib_frame_queue_t *) (~0);
4891 current_worker_index = ~0;
4892 return frame->n_vectors;
4896 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
4897 vlib_node_runtime_t * node,
4898 vlib_frame_t * frame)
4900 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
4903 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
4904 .function = snat_in2out_worker_handoff_fn,
4905 .name = "nat44-in2out-worker-handoff",
4906 .vector_size = sizeof (u32),
4907 .format_trace = format_snat_in2out_worker_handoff_trace,
4908 .type = VLIB_NODE_TYPE_INTERNAL,
4910 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4911 .error_strings = snat_in2out_error_strings,
4920 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
4921 snat_in2out_worker_handoff_fn);
4924 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
4925 vlib_node_runtime_t * node,
4926 vlib_frame_t * frame)
4928 return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
4931 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
4932 .function = snat_in2out_output_worker_handoff_fn,
4933 .name = "nat44-in2out-output-worker-handoff",
4934 .vector_size = sizeof (u32),
4935 .format_trace = format_snat_in2out_worker_handoff_trace,
4936 .type = VLIB_NODE_TYPE_INTERNAL,
4945 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
4946 snat_in2out_output_worker_handoff_fn);
4948 static_always_inline int
4949 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
4951 snat_address_t * ap;
4952 clib_bihash_kv_8_8_t kv, value;
4953 snat_session_key_t m_key;
4955 vec_foreach (ap, sm->addresses)
4957 if (ap->addr.as_u32 == dst_addr->as_u32)
4961 m_key.addr.as_u32 = dst_addr->as_u32;
4962 m_key.fib_index = 0;
4965 kv.key = m_key.as_u64;
4966 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4973 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
4974 vlib_node_runtime_t * node,
4975 vlib_frame_t * frame,
4978 u32 n_left_from, * from, * to_next, stats_node_index;
4979 snat_in2out_next_t next_index;
4980 u32 pkts_processed = 0;
4981 snat_main_t * sm = &snat_main;
4983 stats_node_index = is_ed ? nat44_ed_hairpin_dst_node.index :
4984 snat_hairpin_dst_node.index;
4986 from = vlib_frame_vector_args (frame);
4987 n_left_from = frame->n_vectors;
4988 next_index = node->cached_next_index;
4990 while (n_left_from > 0)
4994 vlib_get_next_frame (vm, node, next_index,
4995 to_next, n_left_to_next);
4997 while (n_left_from > 0 && n_left_to_next > 0)
5005 /* speculatively enqueue b0 to the current next frame */
5011 n_left_to_next -= 1;
5013 b0 = vlib_get_buffer (vm, bi0);
5014 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5015 ip0 = vlib_buffer_get_current (b0);
5017 proto0 = ip_proto_to_snat_proto (ip0->protocol);
5019 vnet_buffer (b0)->snat.flags = 0;
5020 if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
5022 if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
5024 udp_header_t * udp0 = ip4_next_header (ip0);
5025 tcp_header_t * tcp0 = (tcp_header_t *) udp0;
5027 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
5029 else if (proto0 == SNAT_PROTOCOL_ICMP)
5031 icmp46_header_t * icmp0 = ip4_next_header (ip0);
5033 snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
5038 nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
5040 nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
5043 vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
5046 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5048 /* verify speculative enqueue, maybe switch current next frame */
5049 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5050 to_next, n_left_to_next,
5054 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5057 vlib_node_increment_counter (vm, stats_node_index,
5058 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5060 return frame->n_vectors;
5064 snat_hairpin_dst_fn (vlib_main_t * vm,
5065 vlib_node_runtime_t * node,
5066 vlib_frame_t * frame)
5068 return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
5071 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
5072 .function = snat_hairpin_dst_fn,
5073 .name = "nat44-hairpin-dst",
5074 .vector_size = sizeof (u32),
5075 .type = VLIB_NODE_TYPE_INTERNAL,
5076 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5077 .error_strings = snat_in2out_error_strings,
5080 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5081 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5085 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
5086 snat_hairpin_dst_fn);
5089 nat44_ed_hairpin_dst_fn (vlib_main_t * vm,
5090 vlib_node_runtime_t * node,
5091 vlib_frame_t * frame)
5093 return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
5096 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
5097 .function = nat44_ed_hairpin_dst_fn,
5098 .name = "nat44-ed-hairpin-dst",
5099 .vector_size = sizeof (u32),
5100 .type = VLIB_NODE_TYPE_INTERNAL,
5101 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5102 .error_strings = snat_in2out_error_strings,
5105 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5106 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5110 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_dst_node,
5111 nat44_ed_hairpin_dst_fn);
5114 snat_hairpin_src_fn_inline (vlib_main_t * vm,
5115 vlib_node_runtime_t * node,
5116 vlib_frame_t * frame,
5119 u32 n_left_from, * from, * to_next, stats_node_index;
5120 snat_in2out_next_t next_index;
5121 u32 pkts_processed = 0;
5122 snat_main_t *sm = &snat_main;
5124 stats_node_index = is_ed ? nat44_ed_hairpin_src_node.index :
5125 snat_hairpin_src_node.index;
5127 from = vlib_frame_vector_args (frame);
5128 n_left_from = frame->n_vectors;
5129 next_index = node->cached_next_index;
5131 while (n_left_from > 0)
5135 vlib_get_next_frame (vm, node, next_index,
5136 to_next, n_left_to_next);
5138 while (n_left_from > 0 && n_left_to_next > 0)
5143 snat_interface_t *i;
5146 /* speculatively enqueue b0 to the current next frame */
5152 n_left_to_next -= 1;
5154 b0 = vlib_get_buffer (vm, bi0);
5155 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5156 next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
5158 pool_foreach (i, sm->output_feature_interfaces,
5160 /* Only packets from NAT inside interface */
5161 if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
5163 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
5164 SNAT_FLAG_HAIRPINNING))
5166 if (PREDICT_TRUE (sm->num_workers > 1))
5167 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
5169 next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
5175 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5177 /* verify speculative enqueue, maybe switch current next frame */
5178 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5179 to_next, n_left_to_next,
5183 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5186 vlib_node_increment_counter (vm, stats_node_index,
5187 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5189 return frame->n_vectors;
5193 snat_hairpin_src_fn (vlib_main_t * vm,
5194 vlib_node_runtime_t * node,
5195 vlib_frame_t * frame)
5197 return snat_hairpin_src_fn_inline (vm, node, frame, 0);
5200 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
5201 .function = snat_hairpin_src_fn,
5202 .name = "nat44-hairpin-src",
5203 .vector_size = sizeof (u32),
5204 .type = VLIB_NODE_TYPE_INTERNAL,
5205 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5206 .error_strings = snat_in2out_error_strings,
5207 .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5209 [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5210 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
5211 [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5212 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5216 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
5217 snat_hairpin_src_fn);
5220 nat44_ed_hairpin_src_fn (vlib_main_t * vm,
5221 vlib_node_runtime_t * node,
5222 vlib_frame_t * frame)
5224 return snat_hairpin_src_fn_inline (vm, node, frame, 1);
5227 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
5228 .function = nat44_ed_hairpin_src_fn,
5229 .name = "nat44-ed-hairpin-src",
5230 .vector_size = sizeof (u32),
5231 .type = VLIB_NODE_TYPE_INTERNAL,
5232 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5233 .error_strings = snat_in2out_error_strings,
5234 .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5236 [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5237 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
5238 [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5239 [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5243 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_src_node,
5244 nat44_ed_hairpin_src_fn);
5247 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
5248 vlib_node_runtime_t * node,
5249 vlib_frame_t * frame)
5251 u32 n_left_from, * from, * to_next;
5252 snat_in2out_next_t next_index;
5253 u32 pkts_processed = 0;
5254 snat_main_t * sm = &snat_main;
5255 u32 stats_node_index;
5257 stats_node_index = snat_in2out_fast_node.index;
5259 from = vlib_frame_vector_args (frame);
5260 n_left_from = frame->n_vectors;
5261 next_index = node->cached_next_index;
5263 while (n_left_from > 0)
5267 vlib_get_next_frame (vm, node, next_index,
5268 to_next, n_left_to_next);
5270 while (n_left_from > 0 && n_left_to_next > 0)
5278 u32 new_addr0, old_addr0;
5279 u16 old_port0, new_port0;
5280 udp_header_t * udp0;
5281 tcp_header_t * tcp0;
5282 icmp46_header_t * icmp0;
5283 snat_session_key_t key0, sm0;
5287 /* speculatively enqueue b0 to the current next frame */
5293 n_left_to_next -= 1;
5295 b0 = vlib_get_buffer (vm, bi0);
5296 next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5298 ip0 = vlib_buffer_get_current (b0);
5299 udp0 = ip4_next_header (ip0);
5300 tcp0 = (tcp_header_t *) udp0;
5301 icmp0 = (icmp46_header_t *) udp0;
5303 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5304 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
5306 if (PREDICT_FALSE(ip0->ttl == 1))
5308 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
5309 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
5310 ICMP4_time_exceeded_ttl_exceeded_in_transit,
5312 next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
5316 proto0 = ip_proto_to_snat_proto (ip0->protocol);
5318 if (PREDICT_FALSE (proto0 == ~0))
5321 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
5323 next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
5324 rx_fib_index0, node, next0, ~0, 0, 0);
5328 key0.addr = ip0->src_address;
5329 key0.protocol = proto0;
5330 key0.port = udp0->src_port;
5331 key0.fib_index = rx_fib_index0;
5333 if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0, 0))
5335 b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
5336 next0= SNAT_IN2OUT_NEXT_DROP;
5340 new_addr0 = sm0.addr.as_u32;
5341 new_port0 = sm0.port;
5342 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
5343 old_addr0 = ip0->src_address.as_u32;
5344 ip0->src_address.as_u32 = new_addr0;
5346 sum0 = ip0->checksum;
5347 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5349 src_address /* changed member */);
5350 ip0->checksum = ip_csum_fold (sum0);
5352 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
5354 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5356 old_port0 = tcp0->src_port;
5357 tcp0->src_port = new_port0;
5359 sum0 = tcp0->checksum;
5360 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5362 dst_address /* changed member */);
5363 sum0 = ip_csum_update (sum0, old_port0, new_port0,
5364 ip4_header_t /* cheat */,
5365 length /* changed member */);
5366 mss_clamping (sm, tcp0, &sum0);
5367 tcp0->checksum = ip_csum_fold(sum0);
5371 old_port0 = udp0->src_port;
5372 udp0->src_port = new_port0;
5378 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5380 sum0 = tcp0->checksum;
5381 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5383 dst_address /* changed member */);
5384 mss_clamping (sm, tcp0, &sum0);
5385 tcp0->checksum = ip_csum_fold(sum0);
5390 snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
5393 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
5394 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
5396 snat_in2out_trace_t *t =
5397 vlib_add_trace (vm, node, b0, sizeof (*t));
5398 t->sw_if_index = sw_if_index0;
5399 t->next_index = next0;
5402 pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5404 /* verify speculative enqueue, maybe switch current next frame */
5405 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5406 to_next, n_left_to_next,
5410 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5413 vlib_node_increment_counter (vm, stats_node_index,
5414 SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5416 return frame->n_vectors;
5420 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
5421 .function = snat_in2out_fast_static_map_fn,
5422 .name = "nat44-in2out-fast",
5423 .vector_size = sizeof (u32),
5424 .format_trace = format_snat_in2out_fast_trace,
5425 .type = VLIB_NODE_TYPE_INTERNAL,
5427 .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5428 .error_strings = snat_in2out_error_strings,
5430 .runtime_data_bytes = sizeof (snat_runtime_t),
5432 .n_next_nodes = SNAT_IN2OUT_N_NEXT,
5434 /* edit / add dispositions here */
5436 [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5437 [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5438 [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
5439 [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
5440 [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
5444 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);