2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
17 * @brief NAT44 endpoint-dependent outside to inside network translation
20 #include <vlib/vlib.h>
21 #include <vnet/vnet.h>
22 #include <vnet/pg/pg.h>
23 #include <vnet/ip/ip.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/fib/ip4_fib.h>
26 #include <vnet/udp/udp.h>
27 #include <vppinfra/error.h>
29 #include <nat/nat_ipfix_logging.h>
30 #include <nat/nat_inlines.h>
31 #include <nat/nat44/inlines.h>
32 #include <nat/nat_syslog.h>
33 #include <nat/nat_ha.h>
34 #include <nat/nat44/ed_inlines.h>
36 static char *nat_out2in_ed_error_strings[] = {
37 #define _(sym,string) string,
38 foreach_nat_out2in_ed_error
48 } nat44_ed_out2in_trace_t;
51 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
53 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
54 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
55 nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
59 t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
60 "NAT44_OUT2IN_ED_FAST_PATH";
62 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
63 t->sw_if_index, t->next_index, t->session_index);
69 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
70 ip4_header_t * ip0, icmp46_header_t * icmp0,
71 u32 sw_if_index0, u32 rx_fib_index0,
72 vlib_node_runtime_t * node, u32 next0, f64 now,
73 u32 thread_index, snat_session_t ** p_s0)
75 vlib_main_t *vm = vlib_get_main ();
77 next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
78 next0, thread_index, p_s0, 0);
79 snat_session_t *s0 = *p_s0;
80 if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
83 nat44_session_update_counters (s0, now,
84 vlib_buffer_length_in_chain
85 (vm, b0), thread_index);
86 /* Per-user LRU list maintenance */
87 nat44_session_update_lru (sm, s0, thread_index);
92 #ifndef CLIB_MARCH_VARIANT
94 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
96 snat_main_t *sm = &snat_main;
97 nat44_is_idle_session_ctx_t *ctx = arg;
99 u64 sess_timeout_time;
102 ip4_address_t *l_addr, *r_addr;
104 clib_bihash_kv_16_8_t ed_kv;
107 snat_session_key_t key;
108 snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
111 s = pool_elt_at_index (tsm->sessions, kv->value);
112 sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
113 if (ctx->now >= sess_timeout_time)
115 l_addr = &s->in2out.addr;
116 r_addr = &s->ext_host_addr;
117 fib_index = s->in2out.fib_index;
118 if (snat_is_unk_proto_session (s))
120 proto = s->in2out.port;
126 proto = nat_proto_to_ip_proto (s->in2out.protocol);
127 l_port = s->in2out.port;
128 r_port = s->ext_host_port;
130 if (is_twice_nat_session (s))
132 r_addr = &s->ext_host_nat_addr;
133 r_port = s->ext_host_nat_port;
135 make_ed_kv (l_addr, r_addr, proto, fib_index, l_port, r_port, ~0, ~0,
137 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
138 nat_elog_warn ("in2out_ed key del failed");
140 if (snat_is_unk_proto_session (s))
143 snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
144 s->in2out.addr.as_u32,
145 s->out2in.addr.as_u32,
149 s->in2out.fib_index);
151 nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
152 &s->in2out.addr, s->in2out.port,
153 &s->ext_host_nat_addr, s->ext_host_nat_port,
154 &s->out2in.addr, s->out2in.port,
155 &s->ext_host_addr, s->ext_host_port,
156 s->in2out.protocol, is_twice_nat_session (s));
158 nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
159 s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
162 if (is_twice_nat_session (s))
164 for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
166 key.protocol = s->in2out.protocol;
167 key.port = s->ext_host_nat_port;
168 a = sm->twice_nat_addresses + i;
169 if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
171 snat_free_outside_address_and_port (sm->twice_nat_addresses,
179 if (snat_is_session_static (s))
182 snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
185 nat_ed_session_delete (sm, s, ctx->thread_index, 1);
193 static snat_session_t *
194 create_session_for_static_mapping_ed (snat_main_t * sm,
196 snat_session_key_t l_key,
197 snat_session_key_t e_key,
198 vlib_node_runtime_t * node,
201 twice_nat_type_t twice_nat,
202 lb_nat_type_t lb_nat, f64 now)
207 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
208 clib_bihash_kv_16_8_t kv;
209 snat_session_key_t eh_key;
210 nat44_is_idle_session_ctx_t ctx;
213 (nat44_ed_maximum_sessions_exceeded (sm, rx_fib_index, thread_index)))
215 b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
216 nat_elog_notice ("maximum sessions exceeded");
220 s = nat_ed_session_alloc (sm, thread_index, now, e_key.protocol);
223 b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
224 nat_elog_warn ("create NAT session failed");
228 ip = vlib_buffer_get_current (b);
229 udp = ip4_next_header (ip);
231 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
232 s->ext_host_port = e_key.protocol == NAT_PROTOCOL_ICMP ? 0 : udp->src_port;
233 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
235 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
236 if (lb_nat == AFFINITY_LB_NAT)
237 s->flags |= SNAT_SESSION_FLAG_AFFINITY;
238 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
241 s->in2out.protocol = s->out2in.protocol;
243 /* Add to lookup tables */
244 make_ed_kv (&e_key.addr, &s->ext_host_addr, ip->protocol,
245 e_key.fib_index, e_key.port, s->ext_host_port, thread_index,
246 s - tsm->sessions, &kv);
248 ctx.thread_index = thread_index;
249 if (clib_bihash_add_or_overwrite_stale_16_8 (&sm->out2in_ed, &kv,
250 nat44_o2i_ed_is_idle_session_cb,
252 nat_elog_notice ("out2in-ed key add failed");
254 if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
255 ip->src_address.as_u32 == l_key.addr.as_u32))
257 eh_key.protocol = e_key.protocol;
258 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
259 thread_index, &eh_key,
261 tsm->snat_thread_index))
263 b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
264 nat_ed_session_delete (sm, s, thread_index, 1);
265 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &kv, 0))
266 nat_elog_notice ("out2in-ed key del failed");
269 s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
270 s->ext_host_nat_port = eh_key.port;
271 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
272 make_ed_kv (&l_key.addr, &s->ext_host_nat_addr, ip->protocol,
273 l_key.fib_index, l_key.port, s->ext_host_nat_port,
274 thread_index, s - tsm->sessions, &kv);
278 make_ed_kv (&l_key.addr, &s->ext_host_addr, ip->protocol,
279 l_key.fib_index, l_key.port, s->ext_host_port, thread_index,
280 s - tsm->sessions, &kv);
282 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
283 nat44_i2o_ed_is_idle_session_cb,
285 nat_elog_notice ("in2out-ed key add failed");
287 snat_ipfix_logging_nat44_ses_create (thread_index,
288 s->in2out.addr.as_u32,
289 s->out2in.addr.as_u32,
292 s->out2in.port, s->in2out.fib_index);
294 nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
295 &s->in2out.addr, s->in2out.port,
296 &s->ext_host_nat_addr, s->ext_host_nat_port,
297 &s->out2in.addr, s->out2in.port,
298 &s->ext_host_addr, s->ext_host_port,
299 s->in2out.protocol, is_twice_nat_session (s));
301 nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
302 s->out2in.port, &s->ext_host_addr, s->ext_host_port,
303 &s->ext_host_nat_addr, s->ext_host_nat_port,
304 s->in2out.protocol, s->in2out.fib_index, s->flags,
311 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u16 src_port,
312 u16 dst_port, u32 thread_index, u32 rx_fib_index)
314 clib_bihash_kv_16_8_t kv, value;
315 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
317 make_ed_kv (&ip->src_address, &ip->dst_address, ip->protocol,
318 rx_fib_index, src_port, dst_port, ~0, ~0, &kv);
319 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
326 create_bypass_for_fwd (snat_main_t * sm, vlib_buffer_t * b, ip4_header_t * ip,
327 u32 rx_fib_index, u32 thread_index)
329 clib_bihash_kv_16_8_t kv, value;
331 snat_session_t *s = 0;
332 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
333 vlib_main_t *vm = vlib_get_main ();
334 f64 now = vlib_time_now (vm);
337 if (ip->protocol == IP_PROTOCOL_ICMP)
339 if (get_icmp_o2i_ed_key
340 (b, ip, rx_fib_index, ~0, ~0, 0, &l_port, &r_port, &kv))
345 if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
347 udp = ip4_next_header (ip);
348 l_port = udp->dst_port;
349 r_port = udp->src_port;
356 make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
357 rx_fib_index, l_port, r_port, ~0, ~0, &kv);
360 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
362 ASSERT (thread_index == ed_value_get_thread_index (&value));
364 pool_elt_at_index (tsm->sessions,
365 ed_value_get_session_index (&value));
372 (nat44_ed_maximum_sessions_exceeded
373 (sm, rx_fib_index, thread_index)))
376 s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
379 nat_elog_warn ("create NAT session failed");
383 proto = ip_proto_to_nat_proto (ip->protocol);
385 s->ext_host_addr = ip->src_address;
386 s->ext_host_port = r_port;
387 s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
388 s->out2in.addr = ip->dst_address;
389 s->out2in.port = l_port;
390 s->out2in.protocol = proto;
391 if (proto == NAT_PROTOCOL_OTHER)
393 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
394 s->out2in.port = ip->protocol;
396 s->out2in.fib_index = 0;
397 s->in2out = s->out2in;
399 kv.value = s - tsm->sessions;
400 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
401 nat_elog_notice ("in2out_ed key add failed");
404 if (ip->protocol == IP_PROTOCOL_TCP)
406 tcp_header_t *tcp = ip4_next_header (ip);
407 if (nat44_set_tcp_session_state_o2i
408 (sm, now, s, tcp->flags, tcp->ack_number, tcp->seq_number,
414 nat44_session_update_counters (s, now, 0, thread_index);
415 /* Per-user LRU list maintenance */
416 nat44_session_update_lru (sm, s, thread_index);
420 create_bypass_for_fwd_worker (snat_main_t * sm, vlib_buffer_t * b,
421 ip4_header_t * ip, u32 rx_fib_index)
423 ip4_header_t ip_wkr = {
424 .src_address = ip->dst_address,
426 u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
428 create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
431 #ifndef CLIB_MARCH_VARIANT
433 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
434 u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
435 u8 * p_proto, snat_session_key_t * p_value,
436 u8 * p_dont_translate, void *d, void *e)
438 u32 next = ~0, sw_if_index, rx_fib_index;
439 clib_bihash_kv_16_8_t kv, value;
440 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
441 snat_session_t *s = 0;
442 u8 dont_translate = 0, is_addr_only, identity_nat;
443 snat_session_key_t e_key, l_key;
445 vlib_main_t *vm = vlib_get_main ();
447 sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
448 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
450 if (get_icmp_o2i_ed_key
451 (b, ip, rx_fib_index, ~0, ~0, p_proto, &l_port, &r_port, &kv))
453 b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
454 next = NAT_NEXT_DROP;
458 if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv, &value))
460 /* Try to match static mapping */
461 e_key.addr = ip->dst_address;
463 e_key.protocol = ip_proto_to_nat_proto (ip->protocol);
464 e_key.fib_index = rx_fib_index;
465 if (snat_static_mapping_match
466 (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0, &identity_nat))
468 if (!sm->forwarding_enabled)
470 /* Don't NAT packet aimed at the intfc address */
471 if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
472 ip->dst_address.as_u32)))
477 b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
478 next = NAT_NEXT_DROP;
484 if (next_src_nat (sm, ip, l_port, r_port,
485 thread_index, rx_fib_index))
487 next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
490 if (sm->num_workers > 1)
491 create_bypass_for_fwd_worker (sm, b, ip, rx_fib_index);
493 create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
499 (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
501 && (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
502 ICMP4_echo_request || !is_addr_only)))
504 b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
505 next = NAT_NEXT_DROP;
509 if (PREDICT_FALSE (identity_nat))
515 /* Create session initiated by host from external network */
516 s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
517 rx_fib_index, thread_index, 0,
518 0, vlib_time_now (vm));
522 next = NAT_NEXT_DROP;
529 (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
531 && vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags !=
533 && !icmp_type_is_error_message (vnet_buffer (b)->ip.
534 reass.icmp_type_or_tcp_flags)))
536 b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
537 next = NAT_NEXT_DROP;
541 ASSERT (thread_index == ed_value_get_thread_index (&value));
543 pool_elt_at_index (tsm->sessions,
544 ed_value_get_session_index (&value));
548 *p_value = s->in2out;
549 *p_dont_translate = dont_translate;
551 *(snat_session_t **) d = s;
556 static snat_session_t *
557 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
563 vlib_main_t * vm, vlib_node_runtime_t * node)
565 clib_bihash_kv_8_8_t kv, value;
566 clib_bihash_kv_16_8_t s_kv, s_value;
567 snat_static_mapping_t *m;
568 u32 old_addr, new_addr;
571 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
573 old_addr = ip->dst_address.as_u32;
575 make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol, rx_fib_index,
576 0, 0, ~0, ~0, &s_kv);
578 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
580 ASSERT (thread_index == ed_value_get_thread_index (&s_value));
582 pool_elt_at_index (tsm->sessions,
583 ed_value_get_session_index (&s_value));
584 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
589 (nat44_ed_maximum_sessions_exceeded
590 (sm, rx_fib_index, thread_index)))
592 b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
593 nat_elog_notice ("maximum sessions exceeded");
597 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
598 if (clib_bihash_search_8_8
599 (&sm->static_mapping_by_external, &kv, &value))
601 b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
605 m = pool_elt_at_index (sm->static_mappings, value.value);
607 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
609 /* Create a new session */
610 s = nat_ed_session_alloc (sm, thread_index, now, ip->protocol);
613 b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_USER_SESS_EXCEEDED];
614 nat_elog_warn ("create NAT session failed");
618 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
619 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
620 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
621 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
622 s->out2in.addr.as_u32 = old_addr;
623 s->out2in.fib_index = rx_fib_index;
624 s->in2out.addr.as_u32 = new_addr;
625 s->in2out.fib_index = m->fib_index;
626 s->in2out.port = s->out2in.port = ip->protocol;
628 /* Add to lookup tables */
629 s_kv.value = s - tsm->sessions;
630 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
631 nat_elog_notice ("out2in key add failed");
633 make_ed_kv (&ip->dst_address, &ip->src_address, ip->protocol,
634 m->fib_index, 0, 0, thread_index, s - tsm->sessions, &s_kv);
635 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
636 nat_elog_notice ("in2out key add failed");
639 /* Update IP checksum */
641 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
642 ip->checksum = ip_csum_fold (sum);
644 vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
647 nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
649 /* Per-user LRU list maintenance */
650 nat44_session_update_lru (sm, s, thread_index);
656 nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
657 vlib_node_runtime_t * node,
658 vlib_frame_t * frame,
661 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
662 nat_next_t next_index;
663 snat_main_t *sm = &snat_main;
664 f64 now = vlib_time_now (vm);
665 u32 thread_index = vm->thread_index;
666 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
667 u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
670 stats_node_index = sm->ed_out2in_node_index;
672 from = vlib_frame_vector_args (frame);
673 n_left_from = frame->n_vectors;
674 next_index = node->cached_next_index;
676 while (n_left_from > 0)
680 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
682 while (n_left_from > 0 && n_left_to_next > 0)
686 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
688 u16 old_port0, new_port0;
692 snat_session_t *s0 = 0;
693 clib_bihash_kv_16_8_t kv0, value0;
696 /* speculatively enqueue b0 to the current next frame */
704 b0 = vlib_get_buffer (vm, bi0);
705 next0 = vnet_buffer2 (b0)->nat.arc_next;
707 vnet_buffer (b0)->snat.flags = 0;
708 ip0 = vlib_buffer_get_current (b0);
710 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
712 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
715 if (PREDICT_FALSE (ip0->ttl == 1))
717 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
718 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
719 ICMP4_time_exceeded_ttl_exceeded_in_transit,
721 next0 = NAT_NEXT_ICMP_ERROR;
725 udp0 = ip4_next_header (ip0);
726 tcp0 = (tcp_header_t *) udp0;
727 proto0 = ip_proto_to_nat_proto (ip0->protocol);
729 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
731 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
735 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
737 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
741 make_ed_kv (&ip0->dst_address, &ip0->src_address,
742 ip0->protocol, rx_fib_index0,
743 vnet_buffer (b0)->ip.reass.l4_dst_port,
744 vnet_buffer (b0)->ip.reass.l4_src_port, ~0, ~0, &kv0);
746 /* there is a stashed index in vnet_buffer2 from handoff node,
747 * see if we can use it */
748 if (is_multi_worker && PREDICT_TRUE
751 vnet_buffer2 (b0)->nat.ed_out2in_nat_session_index)))
753 s0 = pool_elt_at_index (tsm->sessions,
755 nat.ed_out2in_nat_session_index);
757 (s0->out2in.addr.as_u32 == ip0->dst_address.as_u32
758 && s0->out2in.port ==
759 vnet_buffer (b0)->ip.reass.l4_dst_port
760 && s0->out2in.protocol ==
761 ip_proto_to_nat_proto (ip0->protocol)
762 && s0->out2in.fib_index == rx_fib_index0
763 && s0->ext_host_addr.as_u32 == ip0->src_address.as_u32
764 && s0->ext_host_port ==
765 vnet_buffer (b0)->ip.reass.l4_src_port))
767 /* yes, this is the droid we're looking for */
772 if (clib_bihash_search_16_8 (&sm->out2in_ed, &kv0, &value0))
774 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
777 ASSERT (thread_index == ed_value_get_thread_index (&value0));
779 pool_elt_at_index (tsm->sessions,
780 ed_value_get_session_index (&value0));
783 if (s0->tcp_closed_timestamp)
785 if (now >= s0->tcp_closed_timestamp)
787 // session is closed, go slow path
788 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
792 // session in transitory timeout, drop
793 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_TCP_CLOSED];
794 next0 = NAT_NEXT_DROP;
799 // drop if session expired
800 u64 sess_timeout_time;
801 sess_timeout_time = s0->last_heard +
802 (f64) nat44_session_get_timeout (sm, s0);
803 if (now >= sess_timeout_time)
805 // session is closed, go slow path
806 nat_free_session_data (sm, s0, thread_index, 0);
807 nat_ed_session_delete (sm, s0, thread_index, 1);
808 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
813 old_addr0 = ip0->dst_address.as_u32;
814 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
815 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
817 sum0 = ip0->checksum;
818 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
820 if (PREDICT_FALSE (is_twice_nat_session (s0)))
821 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
822 s0->ext_host_nat_addr.as_u32, ip4_header_t,
824 ip0->checksum = ip_csum_fold (sum0);
826 old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
828 if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
830 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
832 new_port0 = udp0->dst_port = s0->in2out.port;
833 sum0 = tcp0->checksum;
835 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
838 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
840 if (is_twice_nat_session (s0))
842 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
843 s0->ext_host_nat_addr.as_u32,
844 ip4_header_t, dst_address);
846 ip_csum_update (sum0,
847 vnet_buffer (b0)->ip.
849 s0->ext_host_nat_port, ip4_header_t,
851 tcp0->src_port = s0->ext_host_nat_port;
852 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
854 tcp0->checksum = ip_csum_fold (sum0);
857 if (nat44_set_tcp_session_state_o2i
859 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
860 vnet_buffer (b0)->ip.reass.tcp_ack_number,
861 vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
864 else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
867 new_port0 = udp0->dst_port = s0->in2out.port;
868 sum0 = udp0->checksum;
869 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
871 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
873 if (PREDICT_FALSE (is_twice_nat_session (s0)))
875 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
876 s0->ext_host_nat_addr.as_u32,
877 ip4_header_t, dst_address);
879 ip_csum_update (sum0,
880 vnet_buffer (b0)->ip.reass.l4_src_port,
881 s0->ext_host_nat_port, ip4_header_t,
883 udp0->src_port = s0->ext_host_nat_port;
884 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
886 udp0->checksum = ip_csum_fold (sum0);
891 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
893 new_port0 = udp0->dst_port = s0->in2out.port;
894 if (PREDICT_FALSE (is_twice_nat_session (s0)))
896 udp0->src_port = s0->ext_host_nat_port;
897 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
904 nat44_session_update_counters (s0, now,
905 vlib_buffer_length_in_chain (vm, b0),
907 /* Per-user LRU list maintenance */
908 nat44_session_update_lru (sm, s0, thread_index);
911 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
912 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
914 nat44_ed_out2in_trace_t *t =
915 vlib_add_trace (vm, node, b0, sizeof (*t));
916 t->sw_if_index = sw_if_index0;
917 t->next_index = next0;
921 t->session_index = s0 - tsm->sessions;
923 t->session_index = ~0;
926 pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
927 /* verify speculative enqueue, maybe switch current next frame */
928 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
929 to_next, n_left_to_next,
933 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
936 vlib_node_increment_counter (vm, stats_node_index,
937 NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
939 vlib_node_increment_counter (vm, stats_node_index,
940 NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
941 vlib_node_increment_counter (vm, stats_node_index,
942 NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
943 vlib_node_increment_counter (vm, stats_node_index,
944 NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
946 vlib_node_increment_counter (vm, stats_node_index,
947 NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
949 vlib_node_increment_counter (vm, stats_node_index,
950 NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
951 return frame->n_vectors;
955 nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
956 vlib_node_runtime_t * node,
957 vlib_frame_t * frame)
959 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
960 nat_next_t next_index;
961 snat_main_t *sm = &snat_main;
963 f64 now = vlib_time_now (vm);
964 u32 thread_index = vm->thread_index;
965 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
966 u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
969 stats_node_index = sm->ed_out2in_slowpath_node_index;
971 from = vlib_frame_vector_args (frame);
972 n_left_from = frame->n_vectors;
973 next_index = node->cached_next_index;
975 while (n_left_from > 0)
979 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
981 while (n_left_from > 0 && n_left_to_next > 0)
985 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
987 u16 old_port0, new_port0;
991 icmp46_header_t *icmp0;
992 snat_session_t *s0 = 0;
993 clib_bihash_kv_16_8_t kv0, value0;
995 snat_session_key_t e_key0, l_key0;
996 lb_nat_type_t lb_nat0;
997 twice_nat_type_t twice_nat0;
1000 /* speculatively enqueue b0 to the current next frame */
1006 n_left_to_next -= 1;
1008 b0 = vlib_get_buffer (vm, bi0);
1009 next0 = vnet_buffer2 (b0)->nat.arc_next;
1011 vnet_buffer (b0)->snat.flags = 0;
1012 ip0 = vlib_buffer_get_current (b0);
1014 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1016 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1019 if (PREDICT_FALSE (ip0->ttl == 1))
1021 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1022 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1023 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1025 next0 = NAT_NEXT_ICMP_ERROR;
1029 udp0 = ip4_next_header (ip0);
1030 tcp0 = (tcp_header_t *) udp0;
1031 icmp0 = (icmp46_header_t *) udp0;
1032 proto0 = ip_proto_to_nat_proto (ip0->protocol);
1034 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1037 nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1038 thread_index, now, vm, node);
1039 if (!sm->forwarding_enabled)
1042 next0 = NAT_NEXT_DROP;
1048 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1050 next0 = icmp_out2in_ed_slow_path
1051 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1052 next0, now, thread_index, &s0);
1057 make_ed_kv (&ip0->dst_address, &ip0->src_address,
1058 ip0->protocol, rx_fib_index0,
1059 vnet_buffer (b0)->ip.reass.l4_dst_port,
1060 vnet_buffer (b0)->ip.reass.l4_src_port, ~0, ~0, &kv0);
1063 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &kv0, &value0))
1065 ASSERT (thread_index == ed_value_get_thread_index (&value0));
1067 pool_elt_at_index (tsm->sessions,
1068 ed_value_get_session_index (&value0));
1070 if (s0->tcp_closed_timestamp && now >= s0->tcp_closed_timestamp)
1072 nat_free_session_data (sm, s0, thread_index, 0);
1073 nat_ed_session_delete (sm, s0, thread_index, 1);
1080 /* Try to match static mapping by external address and port,
1081 destination address and port in packet */
1082 e_key0.addr = ip0->dst_address;
1083 e_key0.port = vnet_buffer (b0)->ip.reass.l4_dst_port;
1084 e_key0.protocol = proto0;
1085 e_key0.fib_index = rx_fib_index0;
1087 if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1088 &twice_nat0, &lb_nat0,
1093 * Send DHCP packets to the ipv4 stack, or we won't
1094 * be able to use dhcp client on the outside interface
1096 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_UDP
1097 && (vnet_buffer (b0)->ip.
1098 reass.l4_dst_port ==
1099 clib_host_to_net_u16
1100 (UDP_DST_PORT_dhcp_to_client))))
1105 if (!sm->forwarding_enabled)
1108 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1109 next0 = NAT_NEXT_DROP;
1114 (sm, ip0, vnet_buffer (b0)->ip.reass.l4_src_port,
1115 vnet_buffer (b0)->ip.reass.l4_dst_port,
1116 thread_index, rx_fib_index0))
1118 next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1121 if (sm->num_workers > 1)
1122 create_bypass_for_fwd_worker (sm, b0, ip0,
1125 create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
1131 if (PREDICT_FALSE (identity_nat0))
1134 if ((proto0 == NAT_PROTOCOL_TCP)
1135 && !tcp_flags_is_init (vnet_buffer (b0)->ip.
1136 reass.icmp_type_or_tcp_flags))
1138 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1139 next0 = NAT_NEXT_DROP;
1143 /* Create session initiated by host from external network */
1144 s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1152 next0 = NAT_NEXT_DROP;
1157 old_addr0 = ip0->dst_address.as_u32;
1158 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1159 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1161 sum0 = ip0->checksum;
1162 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1164 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1165 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1166 s0->ext_host_nat_addr.as_u32, ip4_header_t,
1168 ip0->checksum = ip_csum_fold (sum0);
1170 old_port0 = vnet_buffer (b0)->ip.reass.l4_dst_port;
1172 if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1174 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1176 new_port0 = udp0->dst_port = s0->in2out.port;
1177 sum0 = tcp0->checksum;
1179 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1182 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1184 if (is_twice_nat_session (s0))
1186 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1187 s0->ext_host_nat_addr.as_u32,
1188 ip4_header_t, dst_address);
1190 ip_csum_update (sum0,
1191 vnet_buffer (b0)->ip.
1193 s0->ext_host_nat_port, ip4_header_t,
1195 tcp0->src_port = s0->ext_host_nat_port;
1196 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1198 tcp0->checksum = ip_csum_fold (sum0);
1201 if (nat44_set_tcp_session_state_o2i
1203 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags,
1204 vnet_buffer (b0)->ip.reass.tcp_ack_number,
1205 vnet_buffer (b0)->ip.reass.tcp_seq_number, thread_index))
1208 else if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment
1211 new_port0 = udp0->dst_port = s0->in2out.port;
1212 sum0 = udp0->checksum;
1213 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1215 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1217 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1219 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1220 s0->ext_host_nat_addr.as_u32,
1221 ip4_header_t, dst_address);
1223 ip_csum_update (sum0,
1224 vnet_buffer (b0)->ip.reass.l4_src_port,
1225 s0->ext_host_nat_port, ip4_header_t,
1227 udp0->src_port = s0->ext_host_nat_port;
1228 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1230 udp0->checksum = ip_csum_fold (sum0);
1235 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1237 new_port0 = udp0->dst_port = s0->in2out.port;
1238 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1240 udp0->src_port = s0->ext_host_nat_port;
1241 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1248 nat44_session_update_counters (s0, now,
1249 vlib_buffer_length_in_chain (vm, b0),
1251 /* Per-user LRU list maintenance */
1252 nat44_session_update_lru (sm, s0, thread_index);
1255 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1256 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1258 nat44_ed_out2in_trace_t *t =
1259 vlib_add_trace (vm, node, b0, sizeof (*t));
1260 t->sw_if_index = sw_if_index0;
1261 t->next_index = next0;
1262 t->is_slow_path = 1;
1265 t->session_index = s0 - tsm->sessions;
1267 t->session_index = ~0;
1270 pkts_processed += next0 == vnet_buffer2 (b0)->nat.arc_next;
1271 /* verify speculative enqueue, maybe switch current next frame */
1272 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1273 to_next, n_left_to_next,
1277 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1280 vlib_node_increment_counter (vm, stats_node_index,
1281 NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1283 vlib_node_increment_counter (vm, stats_node_index,
1284 NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
1285 vlib_node_increment_counter (vm, stats_node_index,
1286 NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
1287 vlib_node_increment_counter (vm, stats_node_index,
1288 NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
1290 vlib_node_increment_counter (vm, stats_node_index,
1291 NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
1293 vlib_node_increment_counter (vm, stats_node_index,
1294 NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
1295 return frame->n_vectors;
1298 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1299 vlib_node_runtime_t * node,
1300 vlib_frame_t * frame)
1302 if (snat_main.num_workers > 1)
1304 return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 1);
1308 return nat44_ed_out2in_fast_path_node_fn_inline (vm, node, frame, 0);
1313 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1314 .name = "nat44-ed-out2in",
1315 .vector_size = sizeof (u32),
1316 .sibling_of = "nat-default",
1317 .format_trace = format_nat44_ed_out2in_trace,
1318 .type = VLIB_NODE_TYPE_INTERNAL,
1319 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1320 .error_strings = nat_out2in_ed_error_strings,
1321 .runtime_data_bytes = sizeof (snat_runtime_t),
1325 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1326 vlib_node_runtime_t * node,
1327 vlib_frame_t * frame)
1329 return nat44_ed_out2in_slow_path_node_fn_inline (vm, node, frame);
1333 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1334 .name = "nat44-ed-out2in-slowpath",
1335 .vector_size = sizeof (u32),
1336 .sibling_of = "nat-default",
1337 .format_trace = format_nat44_ed_out2in_trace,
1338 .type = VLIB_NODE_TYPE_INTERNAL,
1339 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1340 .error_strings = nat_out2in_ed_error_strings,
1341 .runtime_data_bytes = sizeof (snat_runtime_t),
1346 format_nat_pre_trace (u8 * s, va_list * args)
1348 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1349 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1350 nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
1351 return format (s, "out2in next_index %d arc_next_index %d", t->next_index,
1355 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
1356 vlib_node_runtime_t * node,
1357 vlib_frame_t * frame)
1359 return nat_pre_node_fn_inline (vm, node, frame,
1360 NAT_NEXT_OUT2IN_ED_FAST_PATH);
1364 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
1365 .name = "nat-pre-out2in",
1366 .vector_size = sizeof (u32),
1367 .sibling_of = "nat-default",
1368 .format_trace = format_nat_pre_trace,
1369 .type = VLIB_NODE_TYPE_INTERNAL,
1375 * fd.io coding-style-patch-verification: ON
1378 * eval: (c-set-style "gnu")