2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
17 * @brief NAT44 endpoint-dependent outside to inside network translation
20 #include <vlib/vlib.h>
21 #include <vnet/vnet.h>
22 #include <vnet/pg/pg.h>
23 #include <vnet/ip/ip.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/fib/ip4_fib.h>
26 #include <vnet/udp/udp.h>
27 #include <vppinfra/error.h>
29 #include <nat/nat_ipfix_logging.h>
30 #include <nat/nat_reass.h>
31 #include <nat/nat_inlines.h>
32 #include <nat/nat_syslog.h>
33 #include <nat/nat_ha.h>
35 #define foreach_nat_out2in_ed_error \
36 _(UNSUPPORTED_PROTOCOL, "unsupported protocol") \
37 _(OUT2IN_PACKETS, "good out2in packets processed") \
38 _(OUT_OF_PORTS, "out of ports") \
39 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
40 _(NO_TRANSLATION, "no translation") \
41 _(MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded") \
42 _(DROP_FRAGMENT, "drop fragment") \
43 _(MAX_REASS, "maximum reassemblies exceeded") \
44 _(MAX_FRAG, "maximum fragments per reassembly exceeded")\
45 _(NON_SYN, "non-SYN packet try to create session") \
46 _(TCP_PACKETS, "TCP packets") \
47 _(UDP_PACKETS, "UDP packets") \
48 _(ICMP_PACKETS, "ICMP packets") \
49 _(OTHER_PACKETS, "other protocol packets") \
50 _(FRAGMENTS, "fragments") \
51 _(CACHED_FRAGMENTS, "cached fragments") \
52 _(PROCESSED_FRAGMENTS, "processed fragments")
56 #define _(sym,str) NAT_OUT2IN_ED_ERROR_##sym,
57 foreach_nat_out2in_ed_error
59 NAT_OUT2IN_ED_N_ERROR,
60 } nat_out2in_ed_error_t;
62 static char *nat_out2in_ed_error_strings[] = {
63 #define _(sym,string) string,
64 foreach_nat_out2in_ed_error
70 NAT44_ED_OUT2IN_NEXT_DROP,
71 NAT44_ED_OUT2IN_NEXT_LOOKUP,
72 NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
73 NAT44_ED_OUT2IN_NEXT_IN2OUT,
74 NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
75 NAT44_ED_OUT2IN_NEXT_REASS,
76 NAT44_ED_OUT2IN_N_NEXT,
77 } nat44_ed_out2in_next_t;
85 } nat44_ed_out2in_trace_t;
88 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
90 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
91 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
92 nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
96 t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
97 "NAT44_OUT2IN_ED_FAST_PATH";
99 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
100 t->sw_if_index, t->next_index, t->session_index);
106 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
107 ip4_header_t * ip0, icmp46_header_t * icmp0,
108 u32 sw_if_index0, u32 rx_fib_index0,
109 vlib_node_runtime_t * node, u32 next0, f64 now,
110 u32 thread_index, snat_session_t ** p_s0)
112 next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
113 next0, thread_index, p_s0, 0);
114 snat_session_t *s0 = *p_s0;
115 if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP && s0))
118 nat44_session_update_counters (s0, now,
119 vlib_buffer_length_in_chain
120 (sm->vlib_main, b0), thread_index);
121 /* Per-user LRU list maintenance */
122 nat44_session_update_lru (sm, s0, thread_index);
127 #ifndef CLIB_MARCH_VARIANT
129 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
131 snat_main_t *sm = &snat_main;
132 nat44_is_idle_session_ctx_t *ctx = arg;
134 u64 sess_timeout_time;
135 nat_ed_ses_key_t ed_key;
136 clib_bihash_kv_16_8_t ed_kv;
139 snat_session_key_t key;
140 snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
143 s = pool_elt_at_index (tsm->sessions, kv->value);
144 sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
145 if (ctx->now >= sess_timeout_time)
147 ed_key.l_addr = s->in2out.addr;
148 ed_key.r_addr = s->ext_host_addr;
149 ed_key.fib_index = s->in2out.fib_index;
150 if (snat_is_unk_proto_session (s))
152 ed_key.proto = s->in2out.port;
158 ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
159 ed_key.l_port = s->in2out.port;
160 ed_key.r_port = s->ext_host_port;
162 if (is_twice_nat_session (s))
164 ed_key.r_addr = s->ext_host_nat_addr;
165 ed_key.r_port = s->ext_host_nat_port;
167 ed_kv.key[0] = ed_key.as_u64[0];
168 ed_kv.key[1] = ed_key.as_u64[1];
169 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
170 nat_log_warn ("in2out_ed key del failed");
172 if (snat_is_unk_proto_session (s))
175 snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
176 s->in2out.addr.as_u32,
177 s->out2in.addr.as_u32,
181 s->in2out.fib_index);
183 nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
184 &s->in2out.addr, s->in2out.port,
185 &s->ext_host_nat_addr, s->ext_host_nat_port,
186 &s->out2in.addr, s->out2in.port,
187 &s->ext_host_addr, s->ext_host_port,
188 s->in2out.protocol, is_twice_nat_session (s));
190 nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
191 s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
194 if (is_twice_nat_session (s))
196 for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
198 key.protocol = s->in2out.protocol;
199 key.port = s->ext_host_nat_port;
200 a = sm->twice_nat_addresses + i;
201 if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
203 snat_free_outside_address_and_port (sm->twice_nat_addresses,
211 if (snat_is_session_static (s))
214 snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
217 nat44_delete_session (sm, s, ctx->thread_index);
225 static snat_session_t *
226 create_session_for_static_mapping_ed (snat_main_t * sm,
228 snat_session_key_t l_key,
229 snat_session_key_t e_key,
230 vlib_node_runtime_t * node,
232 twice_nat_type_t twice_nat,
233 lb_nat_type_t lb_nat, f64 now)
239 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
240 clib_bihash_kv_16_8_t kv;
241 snat_session_key_t eh_key;
242 nat44_is_idle_session_ctx_t ctx;
244 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
246 b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
247 nat_log_notice ("maximum sessions exceeded");
251 u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
254 nat_log_warn ("create NAT user failed");
258 s = nat_ed_session_alloc (sm, u, thread_index, now);
261 nat44_delete_user_with_no_session (sm, u, thread_index);
262 nat_log_warn ("create NAT session failed");
266 ip = vlib_buffer_get_current (b);
267 udp = ip4_next_header (ip);
269 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
270 s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
271 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
273 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
274 if (lb_nat == AFFINITY_LB_NAT)
275 s->flags |= SNAT_SESSION_FLAG_AFFINITY;
276 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
279 s->in2out.protocol = s->out2in.protocol;
280 user_session_increment (sm, u, 1);
282 /* Add to lookup tables */
283 make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
284 e_key.fib_index, e_key.port, s->ext_host_port);
285 kv.value = s - tsm->sessions;
287 ctx.thread_index = thread_index;
288 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
289 nat44_o2i_ed_is_idle_session_cb,
291 nat_log_notice ("out2in-ed key add failed");
293 if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
294 ip->src_address.as_u32 == l_key.addr.as_u32))
296 eh_key.protocol = e_key.protocol;
297 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
298 thread_index, &eh_key,
300 tsm->snat_thread_index))
302 b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
303 nat44_delete_session (sm, s, thread_index);
304 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
305 nat_log_notice ("out2in-ed key del failed");
308 s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
309 s->ext_host_nat_port = eh_key.port;
310 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
311 make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
312 l_key.fib_index, l_key.port, s->ext_host_nat_port);
316 make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
317 l_key.fib_index, l_key.port, s->ext_host_port);
319 kv.value = s - tsm->sessions;
320 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
321 nat44_i2o_ed_is_idle_session_cb,
323 nat_log_notice ("in2out-ed key add failed");
325 snat_ipfix_logging_nat44_ses_create (thread_index,
326 s->in2out.addr.as_u32,
327 s->out2in.addr.as_u32,
330 s->out2in.port, s->in2out.fib_index);
332 nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
333 &s->in2out.addr, s->in2out.port,
334 &s->ext_host_nat_addr, s->ext_host_nat_port,
335 &s->out2in.addr, s->out2in.port,
336 &s->ext_host_addr, s->ext_host_port,
337 s->in2out.protocol, is_twice_nat_session (s));
339 nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
340 s->out2in.port, &s->ext_host_addr, s->ext_host_port,
341 &s->ext_host_nat_addr, s->ext_host_nat_port,
342 s->in2out.protocol, s->in2out.fib_index, s->flags,
348 static_always_inline int
349 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
351 icmp46_header_t *icmp0;
352 nat_ed_ses_key_t key0;
353 icmp_echo_header_t *echo0, *inner_echo0 = 0;
354 ip4_header_t *inner_ip0;
356 icmp46_header_t *inner_icmp0;
358 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
359 echo0 = (icmp_echo_header_t *) (icmp0 + 1);
361 if (!icmp_is_error_message (icmp0))
363 key0.proto = IP_PROTOCOL_ICMP;
364 key0.l_addr = ip0->dst_address;
365 key0.r_addr = ip0->src_address;
366 key0.l_port = echo0->identifier;
371 inner_ip0 = (ip4_header_t *) (echo0 + 1);
372 l4_header = ip4_next_header (inner_ip0);
373 key0.proto = inner_ip0->protocol;
374 key0.l_addr = inner_ip0->src_address;
375 key0.r_addr = inner_ip0->dst_address;
376 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
378 case SNAT_PROTOCOL_ICMP:
379 inner_icmp0 = (icmp46_header_t *) l4_header;
380 inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
381 key0.l_port = inner_echo0->identifier;
384 case SNAT_PROTOCOL_UDP:
385 case SNAT_PROTOCOL_TCP:
386 key0.l_port = ((tcp_udp_header_t *) l4_header)->src_port;
387 key0.r_port = ((tcp_udp_header_t *) l4_header)->dst_port;
398 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
399 u16 dst_port, u32 thread_index, u32 rx_fib_index)
401 clib_bihash_kv_16_8_t kv, value;
402 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
404 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
405 rx_fib_index, src_port, dst_port);
406 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
413 create_bypass_for_fwd (snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
416 nat_ed_ses_key_t key;
417 clib_bihash_kv_16_8_t kv, value;
420 snat_session_t *s = 0;
421 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
422 f64 now = vlib_time_now (sm->vlib_main);
424 if (ip->protocol == IP_PROTOCOL_ICMP)
426 if (icmp_get_ed_key (ip, &key))
429 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
431 udp = ip4_next_header (ip);
432 key.r_addr = ip->src_address;
433 key.l_addr = ip->dst_address;
434 key.proto = ip->protocol;
435 key.l_port = udp->dst_port;
436 key.r_port = udp->src_port;
440 key.r_addr = ip->src_address;
441 key.l_addr = ip->dst_address;
442 key.proto = ip->protocol;
443 key.l_port = key.r_port = 0;
446 kv.key[0] = key.as_u64[0];
447 kv.key[1] = key.as_u64[1];
449 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
451 s = pool_elt_at_index (tsm->sessions, value.value);
457 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
460 u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
464 nat_log_warn ("create NAT user failed");
468 s = nat_ed_session_alloc (sm, u, thread_index, now);
471 nat44_delete_user_with_no_session (sm, u, thread_index);
472 nat_log_warn ("create NAT session failed");
476 proto = ip_proto_to_snat_proto (key.proto);
478 s->ext_host_addr = key.r_addr;
479 s->ext_host_port = key.r_port;
480 s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
481 s->out2in.addr = key.l_addr;
482 s->out2in.port = key.l_port;
483 s->out2in.protocol = proto;
486 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
487 s->out2in.port = ip->protocol;
489 s->out2in.fib_index = 0;
490 s->in2out = s->out2in;
491 user_session_increment (sm, u, 0);
493 kv.value = s - tsm->sessions;
494 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
495 nat_log_notice ("in2out_ed key add failed");
498 if (ip->protocol == IP_PROTOCOL_TCP)
500 tcp_header_t *tcp = ip4_next_header (ip);
501 if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
506 nat44_session_update_counters (s, now, 0, thread_index);
507 /* Per-user LRU list maintenance */
508 nat44_session_update_lru (sm, s, thread_index);
512 create_bypass_for_fwd_worker (snat_main_t * sm, ip4_header_t * ip,
515 ip4_header_t ip_wkr = {
516 .src_address = ip->dst_address,
518 u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index);
520 create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
523 #ifndef CLIB_MARCH_VARIANT
525 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
526 u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
527 u8 * p_proto, snat_session_key_t * p_value,
528 u8 * p_dont_translate, void *d, void *e)
530 u32 next = ~0, sw_if_index, rx_fib_index;
531 icmp46_header_t *icmp;
532 nat_ed_ses_key_t key;
533 clib_bihash_kv_16_8_t kv, value;
534 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
535 snat_session_t *s = 0;
536 u8 dont_translate = 0, is_addr_only, identity_nat;
537 snat_session_key_t e_key, l_key;
539 icmp = (icmp46_header_t *) ip4_next_header (ip);
540 sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
541 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
543 if (icmp_get_ed_key (ip, &key))
545 b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
546 next = NAT44_ED_OUT2IN_NEXT_DROP;
549 key.fib_index = rx_fib_index;
550 kv.key[0] = key.as_u64[0];
551 kv.key[1] = key.as_u64[1];
553 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
555 /* Try to match static mapping */
556 e_key.addr = ip->dst_address;
557 e_key.port = key.l_port;
558 e_key.protocol = ip_proto_to_snat_proto (key.proto);
559 e_key.fib_index = rx_fib_index;
560 if (snat_static_mapping_match
561 (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0, &identity_nat))
563 if (!sm->forwarding_enabled)
565 /* Don't NAT packet aimed at the intfc address */
566 if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
567 ip->dst_address.as_u32)))
572 b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
573 next = NAT44_ED_OUT2IN_NEXT_DROP;
579 if (next_src_nat (sm, ip, key.proto, key.l_port, key.r_port,
580 thread_index, rx_fib_index))
582 next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
585 if (sm->num_workers > 1)
586 create_bypass_for_fwd_worker (sm, ip, rx_fib_index);
588 create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
593 if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
594 (icmp->type != ICMP4_echo_request || !is_addr_only)))
596 b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
597 next = NAT44_ED_OUT2IN_NEXT_DROP;
601 if (PREDICT_FALSE (identity_nat))
607 /* Create session initiated by host from external network */
608 s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
615 next = NAT44_ED_OUT2IN_NEXT_DROP;
621 if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
622 icmp->type != ICMP4_echo_request &&
623 !icmp_is_error_message (icmp)))
625 b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
626 next = NAT44_ED_OUT2IN_NEXT_DROP;
630 s = pool_elt_at_index (tsm->sessions, value.value);
633 *p_proto = ip_proto_to_snat_proto (key.proto);
636 *p_value = s->in2out;
637 *p_dont_translate = dont_translate;
639 *(snat_session_t **) d = s;
644 static snat_session_t *
645 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
651 vlib_main_t * vm, vlib_node_runtime_t * node)
653 clib_bihash_kv_8_8_t kv, value;
654 clib_bihash_kv_16_8_t s_kv, s_value;
655 snat_static_mapping_t *m;
656 u32 old_addr, new_addr;
659 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
662 old_addr = ip->dst_address.as_u32;
664 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
667 if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
669 s = pool_elt_at_index (tsm->sessions, s_value.value);
670 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
674 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
676 b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
677 nat_log_notice ("maximum sessions exceeded");
681 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
682 if (clib_bihash_search_8_8
683 (&sm->static_mapping_by_external, &kv, &value))
685 b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
689 m = pool_elt_at_index (sm->static_mappings, value.value);
691 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
693 u = nat_user_get_or_create (sm, &m->local_addr, m->fib_index,
697 nat_log_warn ("create NAT user failed");
701 /* Create a new session */
702 s = nat_ed_session_alloc (sm, u, thread_index, now);
705 nat44_delete_user_with_no_session (sm, u, thread_index);
706 nat_log_warn ("create NAT session failed");
710 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
711 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
712 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
713 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
714 s->out2in.addr.as_u32 = old_addr;
715 s->out2in.fib_index = rx_fib_index;
716 s->in2out.addr.as_u32 = new_addr;
717 s->in2out.fib_index = m->fib_index;
718 s->in2out.port = s->out2in.port = ip->protocol;
719 user_session_increment (sm, u, 1);
721 /* Add to lookup tables */
722 s_kv.value = s - tsm->sessions;
723 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
724 nat_log_notice ("out2in key add failed");
726 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
728 s_kv.value = s - tsm->sessions;
729 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
730 nat_log_notice ("in2out key add failed");
733 /* Update IP checksum */
735 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
736 ip->checksum = ip_csum_fold (sum);
738 vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
741 nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
743 /* Per-user LRU list maintenance */
744 nat44_session_update_lru (sm, s, thread_index);
750 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
751 vlib_node_runtime_t * node,
752 vlib_frame_t * frame, int is_slow_path)
754 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
755 nat44_ed_out2in_next_t next_index;
756 snat_main_t *sm = &snat_main;
757 f64 now = vlib_time_now (vm);
758 u32 thread_index = vm->thread_index;
759 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
760 u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
763 stats_node_index = is_slow_path ? sm->ed_out2in_slowpath_node_index :
764 sm->ed_out2in_node_index;
766 from = vlib_frame_vector_args (frame);
767 n_left_from = frame->n_vectors;
768 next_index = node->cached_next_index;
770 while (n_left_from > 0)
774 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
776 while (n_left_from >= 4 && n_left_to_next >= 2)
779 vlib_buffer_t *b0, *b1;
780 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
782 u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1,
784 u16 old_port0, new_port0, old_port1, new_port1;
785 ip4_header_t *ip0, *ip1;
786 udp_header_t *udp0, *udp1;
787 tcp_header_t *tcp0, *tcp1;
788 icmp46_header_t *icmp0, *icmp1;
789 snat_session_t *s0 = 0, *s1 = 0;
790 clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
791 ip_csum_t sum0, sum1;
792 snat_session_key_t e_key0, l_key0, e_key1, l_key1;
793 lb_nat_type_t lb_nat0, lb_nat1;
794 twice_nat_type_t twice_nat0, twice_nat1;
795 u8 identity_nat0, identity_nat1;
797 /* Prefetch next iteration. */
799 vlib_buffer_t *p2, *p3;
801 p2 = vlib_get_buffer (vm, from[2]);
802 p3 = vlib_get_buffer (vm, from[3]);
804 vlib_prefetch_buffer_header (p2, LOAD);
805 vlib_prefetch_buffer_header (p3, LOAD);
807 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
808 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
811 /* speculatively enqueue b0 and b1 to the current next frame */
812 to_next[0] = bi0 = from[0];
813 to_next[1] = bi1 = from[1];
819 b0 = vlib_get_buffer (vm, bi0);
820 b1 = vlib_get_buffer (vm, bi1);
822 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
823 vnet_buffer (b0)->snat.flags = 0;
824 ip0 = vlib_buffer_get_current (b0);
826 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
828 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
831 if (PREDICT_FALSE (ip0->ttl == 1))
833 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
834 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
835 ICMP4_time_exceeded_ttl_exceeded_in_transit,
837 next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
841 udp0 = ip4_next_header (ip0);
842 tcp0 = (tcp_header_t *) udp0;
843 icmp0 = (icmp46_header_t *) udp0;
844 proto0 = ip_proto_to_snat_proto (ip0->protocol);
848 if (PREDICT_FALSE (proto0 == ~0))
851 nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
852 thread_index, now, vm,
855 if (!sm->forwarding_enabled)
858 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
863 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
865 next0 = icmp_out2in_ed_slow_path
866 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
867 next0, now, thread_index, &s0);
874 if (PREDICT_FALSE (proto0 == ~0))
876 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
880 if (ip4_is_fragment (ip0))
882 next0 = NAT44_ED_OUT2IN_NEXT_REASS;
887 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
889 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
894 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
895 ip0->protocol, rx_fib_index0, udp0->dst_port,
898 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
902 /* Try to match static mapping by external address and port,
903 destination address and port in packet */
904 e_key0.addr = ip0->dst_address;
905 e_key0.port = udp0->dst_port;
906 e_key0.protocol = proto0;
907 e_key0.fib_index = rx_fib_index0;
908 if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
909 &twice_nat0, &lb_nat0,
914 * Send DHCP packets to the ipv4 stack, or we won't
915 * be able to use dhcp client on the outside interface
917 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
918 && (udp0->dst_port ==
920 (UDP_DST_PORT_dhcp_to_client))))
922 vnet_feature_next (&next0, b0);
926 if (!sm->forwarding_enabled)
929 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
930 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
934 if (next_src_nat (sm, ip0, ip0->protocol,
935 udp0->src_port, udp0->dst_port,
936 thread_index, rx_fib_index0))
938 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
941 if (sm->num_workers > 1)
942 create_bypass_for_fwd_worker (sm, ip0,
945 create_bypass_for_fwd (sm, ip0, rx_fib_index0,
951 if (PREDICT_FALSE (identity_nat0))
954 if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
956 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
957 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
961 /* Create session initiated by host from external network */
962 s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
970 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
976 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
982 s0 = pool_elt_at_index (tsm->sessions, value0.value);
985 old_addr0 = ip0->dst_address.as_u32;
986 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
987 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
989 sum0 = ip0->checksum;
990 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
992 if (PREDICT_FALSE (is_twice_nat_session (s0)))
993 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
994 s0->ext_host_nat_addr.as_u32, ip4_header_t,
996 ip0->checksum = ip_csum_fold (sum0);
998 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1000 old_port0 = tcp0->dst_port;
1001 new_port0 = tcp0->dst_port = s0->in2out.port;
1003 sum0 = tcp0->checksum;
1004 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1006 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1008 if (is_twice_nat_session (s0))
1010 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1011 s0->ext_host_nat_addr.as_u32,
1012 ip4_header_t, dst_address);
1013 sum0 = ip_csum_update (sum0, tcp0->src_port,
1014 s0->ext_host_nat_port, ip4_header_t,
1016 tcp0->src_port = s0->ext_host_nat_port;
1017 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1019 tcp0->checksum = ip_csum_fold (sum0);
1021 if (nat44_set_tcp_session_state_o2i
1022 (sm, s0, tcp0, thread_index))
1027 udp0->dst_port = s0->in2out.port;
1028 if (is_twice_nat_session (s0))
1030 udp0->src_port = s0->ext_host_nat_port;
1031 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1038 nat44_session_update_counters (s0, now,
1039 vlib_buffer_length_in_chain (vm, b0),
1041 /* Per-user LRU list maintenance */
1042 nat44_session_update_lru (sm, s0, thread_index);
1045 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1046 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1048 nat44_ed_out2in_trace_t *t =
1049 vlib_add_trace (vm, node, b0, sizeof (*t));
1050 t->is_slow_path = is_slow_path;
1051 t->sw_if_index = sw_if_index0;
1052 t->next_index = next0;
1053 t->session_index = ~0;
1055 t->session_index = s0 - tsm->sessions;
1058 pkts_processed += next0 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1060 next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1061 vnet_buffer (b1)->snat.flags = 0;
1062 ip1 = vlib_buffer_get_current (b1);
1064 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1066 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1069 if (PREDICT_FALSE (ip1->ttl == 1))
1071 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1072 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1073 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1075 next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1079 udp1 = ip4_next_header (ip1);
1080 tcp1 = (tcp_header_t *) udp1;
1081 icmp1 = (icmp46_header_t *) udp1;
1082 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1086 if (PREDICT_FALSE (proto1 == ~0))
1089 nat44_ed_out2in_unknown_proto (sm, b1, ip1, rx_fib_index1,
1090 thread_index, now, vm,
1093 if (!sm->forwarding_enabled)
1096 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1101 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1103 next1 = icmp_out2in_ed_slow_path
1104 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1105 next1, now, thread_index, &s1);
1112 if (PREDICT_FALSE (proto1 == ~0))
1114 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1118 if (ip4_is_fragment (ip1))
1120 next1 = NAT44_ED_OUT2IN_NEXT_REASS;
1125 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1127 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1132 make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address,
1133 ip1->protocol, rx_fib_index1, udp1->dst_port,
1136 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
1140 /* Try to match static mapping by external address and port,
1141 destination address and port in packet */
1142 e_key1.addr = ip1->dst_address;
1143 e_key1.port = udp1->dst_port;
1144 e_key1.protocol = proto1;
1145 e_key1.fib_index = rx_fib_index1;
1146 if (snat_static_mapping_match (sm, e_key1, &l_key1, 1, 0,
1147 &twice_nat1, &lb_nat1,
1152 * Send DHCP packets to the ipv4 stack, or we won't
1153 * be able to use dhcp client on the outside interface
1155 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1156 && (udp1->dst_port ==
1157 clib_host_to_net_u16
1158 (UDP_DST_PORT_dhcp_to_client))))
1160 vnet_feature_next (&next1, b1);
1164 if (!sm->forwarding_enabled)
1167 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1168 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1172 if (next_src_nat (sm, ip1, ip1->protocol,
1173 udp1->src_port, udp1->dst_port,
1174 thread_index, rx_fib_index1))
1176 next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1179 if (sm->num_workers > 1)
1180 create_bypass_for_fwd_worker (sm, ip1,
1183 create_bypass_for_fwd (sm, ip1, rx_fib_index1,
1189 if (PREDICT_FALSE (identity_nat1))
1192 if ((proto1 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp1))
1194 b1->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1195 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1199 /* Create session initiated by host from external network */
1200 s1 = create_session_for_static_mapping_ed (sm, b1, l_key1,
1208 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1214 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1220 s1 = pool_elt_at_index (tsm->sessions, value1.value);
1223 old_addr1 = ip1->dst_address.as_u32;
1224 new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
1225 vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1227 sum1 = ip1->checksum;
1228 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1230 if (PREDICT_FALSE (is_twice_nat_session (s1)))
1231 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1232 s1->ext_host_nat_addr.as_u32, ip4_header_t,
1234 ip1->checksum = ip_csum_fold (sum1);
1236 if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1238 old_port1 = tcp1->dst_port;
1239 new_port1 = tcp1->dst_port = s1->in2out.port;
1241 sum1 = tcp1->checksum;
1242 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1244 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1246 if (is_twice_nat_session (s1))
1248 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1249 s1->ext_host_nat_addr.as_u32,
1250 ip4_header_t, dst_address);
1251 sum1 = ip_csum_update (sum1, tcp1->src_port,
1252 s1->ext_host_nat_port, ip4_header_t,
1254 tcp1->src_port = s1->ext_host_nat_port;
1255 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1257 tcp1->checksum = ip_csum_fold (sum1);
1259 if (nat44_set_tcp_session_state_o2i
1260 (sm, s1, tcp1, thread_index))
1265 udp1->dst_port = s1->in2out.port;
1266 if (is_twice_nat_session (s1))
1268 udp1->src_port = s1->ext_host_nat_port;
1269 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1276 nat44_session_update_counters (s1, now,
1277 vlib_buffer_length_in_chain (vm, b1),
1279 /* Per-user LRU list maintenance */
1280 nat44_session_update_lru (sm, s1, thread_index);
1283 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1284 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1286 nat44_ed_out2in_trace_t *t =
1287 vlib_add_trace (vm, node, b1, sizeof (*t));
1288 t->is_slow_path = is_slow_path;
1289 t->sw_if_index = sw_if_index1;
1290 t->next_index = next1;
1291 t->session_index = ~0;
1293 t->session_index = s1 - tsm->sessions;
1296 pkts_processed += next1 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1298 /* verify speculative enqueues, maybe switch current next frame */
1299 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1300 to_next, n_left_to_next,
1301 bi0, bi1, next0, next1);
1304 while (n_left_from > 0 && n_left_to_next > 0)
1308 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
1310 u16 old_port0, new_port0;
1314 icmp46_header_t *icmp0;
1315 snat_session_t *s0 = 0;
1316 clib_bihash_kv_16_8_t kv0, value0;
1318 snat_session_key_t e_key0, l_key0;
1319 lb_nat_type_t lb_nat0;
1320 twice_nat_type_t twice_nat0;
1323 /* speculatively enqueue b0 to the current next frame */
1329 n_left_to_next -= 1;
1331 b0 = vlib_get_buffer (vm, bi0);
1332 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1333 vnet_buffer (b0)->snat.flags = 0;
1334 ip0 = vlib_buffer_get_current (b0);
1336 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1338 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1341 if (PREDICT_FALSE (ip0->ttl == 1))
1343 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1344 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1345 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1347 next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1351 udp0 = ip4_next_header (ip0);
1352 tcp0 = (tcp_header_t *) udp0;
1353 icmp0 = (icmp46_header_t *) udp0;
1354 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1358 if (PREDICT_FALSE (proto0 == ~0))
1361 nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1362 thread_index, now, vm,
1365 if (!sm->forwarding_enabled)
1368 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1373 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1375 next0 = icmp_out2in_ed_slow_path
1376 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1377 next0, now, thread_index, &s0);
1384 if (PREDICT_FALSE (proto0 == ~0))
1386 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1390 if (ip4_is_fragment (ip0))
1392 next0 = NAT44_ED_OUT2IN_NEXT_REASS;
1397 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1399 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1404 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1405 ip0->protocol, rx_fib_index0, udp0->dst_port,
1408 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
1412 /* Try to match static mapping by external address and port,
1413 destination address and port in packet */
1414 e_key0.addr = ip0->dst_address;
1415 e_key0.port = udp0->dst_port;
1416 e_key0.protocol = proto0;
1417 e_key0.fib_index = rx_fib_index0;
1418 if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1419 &twice_nat0, &lb_nat0,
1424 * Send DHCP packets to the ipv4 stack, or we won't
1425 * be able to use dhcp client on the outside interface
1427 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1428 && (udp0->dst_port ==
1429 clib_host_to_net_u16
1430 (UDP_DST_PORT_dhcp_to_client))))
1432 vnet_feature_next (&next0, b0);
1436 if (!sm->forwarding_enabled)
1439 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1440 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1444 if (next_src_nat (sm, ip0, ip0->protocol,
1445 udp0->src_port, udp0->dst_port,
1446 thread_index, rx_fib_index0))
1448 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1451 if (sm->num_workers > 1)
1452 create_bypass_for_fwd_worker (sm, ip0,
1455 create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1461 if (PREDICT_FALSE (identity_nat0))
1464 if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1466 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1467 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1471 /* Create session initiated by host from external network */
1472 s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1480 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1486 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1492 s0 = pool_elt_at_index (tsm->sessions, value0.value);
1495 old_addr0 = ip0->dst_address.as_u32;
1496 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1497 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1499 sum0 = ip0->checksum;
1500 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1502 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1503 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1504 s0->ext_host_nat_addr.as_u32, ip4_header_t,
1506 ip0->checksum = ip_csum_fold (sum0);
1508 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1510 old_port0 = tcp0->dst_port;
1511 new_port0 = tcp0->dst_port = s0->in2out.port;
1513 sum0 = tcp0->checksum;
1514 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1516 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1518 if (is_twice_nat_session (s0))
1520 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1521 s0->ext_host_nat_addr.as_u32,
1522 ip4_header_t, dst_address);
1523 sum0 = ip_csum_update (sum0, tcp0->src_port,
1524 s0->ext_host_nat_port, ip4_header_t,
1526 tcp0->src_port = s0->ext_host_nat_port;
1527 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1529 tcp0->checksum = ip_csum_fold (sum0);
1531 if (nat44_set_tcp_session_state_o2i
1532 (sm, s0, tcp0, thread_index))
1537 udp0->dst_port = s0->in2out.port;
1538 if (is_twice_nat_session (s0))
1540 udp0->src_port = s0->ext_host_nat_port;
1541 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1548 nat44_session_update_counters (s0, now,
1549 vlib_buffer_length_in_chain (vm, b0),
1551 /* Per-user LRU list maintenance */
1552 nat44_session_update_lru (sm, s0, thread_index);
1555 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1556 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1558 nat44_ed_out2in_trace_t *t =
1559 vlib_add_trace (vm, node, b0, sizeof (*t));
1560 t->is_slow_path = is_slow_path;
1561 t->sw_if_index = sw_if_index0;
1562 t->next_index = next0;
1563 t->session_index = ~0;
1565 t->session_index = s0 - tsm->sessions;
1568 pkts_processed += next0 == NAT44_ED_OUT2IN_NEXT_LOOKUP;
1569 /* verify speculative enqueue, maybe switch current next frame */
1570 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1571 to_next, n_left_to_next,
1575 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1578 vlib_node_increment_counter (vm, stats_node_index,
1579 NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1581 vlib_node_increment_counter (vm, stats_node_index,
1582 NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
1583 vlib_node_increment_counter (vm, stats_node_index,
1584 NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
1585 vlib_node_increment_counter (vm, stats_node_index,
1586 NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
1588 vlib_node_increment_counter (vm, stats_node_index,
1589 NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
1591 vlib_node_increment_counter (vm, stats_node_index,
1592 NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
1593 return frame->n_vectors;
1596 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1597 vlib_node_runtime_t * node,
1598 vlib_frame_t * frame)
1600 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
1604 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1605 .name = "nat44-ed-out2in",
1606 .vector_size = sizeof (u32),
1607 .format_trace = format_nat44_ed_out2in_trace,
1608 .type = VLIB_NODE_TYPE_INTERNAL,
1609 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1610 .error_strings = nat_out2in_ed_error_strings,
1611 .runtime_data_bytes = sizeof (snat_runtime_t),
1612 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1614 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1615 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1616 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1617 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1618 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1619 [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1624 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1625 vlib_node_runtime_t * node,
1626 vlib_frame_t * frame)
1628 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
1632 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1633 .name = "nat44-ed-out2in-slowpath",
1634 .vector_size = sizeof (u32),
1635 .format_trace = format_nat44_ed_out2in_trace,
1636 .type = VLIB_NODE_TYPE_INTERNAL,
1637 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1638 .error_strings = nat_out2in_ed_error_strings,
1639 .runtime_data_bytes = sizeof (snat_runtime_t),
1640 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1642 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1643 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1644 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1645 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1646 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1647 [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1652 VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm,
1653 vlib_node_runtime_t * node,
1654 vlib_frame_t * frame)
1656 u32 n_left_from, *from, *to_next;
1657 nat44_ed_out2in_next_t next_index;
1658 u32 pkts_processed = 0;
1659 snat_main_t *sm = &snat_main;
1660 f64 now = vlib_time_now (vm);
1661 u32 thread_index = vm->thread_index;
1662 snat_main_per_thread_data_t *per_thread_data =
1663 &sm->per_thread_data[thread_index];
1664 u32 *fragments_to_drop = 0;
1665 u32 *fragments_to_loopback = 0;
1667 from = vlib_frame_vector_args (frame);
1668 n_left_from = frame->n_vectors;
1669 next_index = node->cached_next_index;
1671 while (n_left_from > 0)
1675 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1677 while (n_left_from > 0 && n_left_to_next > 0)
1679 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1684 nat_reass_ip4_t *reass0;
1687 icmp46_header_t *icmp0;
1688 clib_bihash_kv_16_8_t kv0, value0;
1689 snat_session_t *s0 = 0;
1690 u16 old_port0, new_port0;
1692 snat_session_key_t e_key0, l_key0;
1694 twice_nat_type_t twice_nat0;
1697 /* speculatively enqueue b0 to the current next frame */
1703 n_left_to_next -= 1;
1705 b0 = vlib_get_buffer (vm, bi0);
1706 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1708 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1710 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1713 if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1715 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1716 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT];
1720 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1721 udp0 = ip4_next_header (ip0);
1722 tcp0 = (tcp_header_t *) udp0;
1723 icmp0 = (icmp46_header_t *) udp0;
1724 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1726 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1730 1, &fragments_to_drop);
1732 if (PREDICT_FALSE (!reass0))
1734 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1735 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_REASS];
1736 nat_log_notice ("maximum reassemblies exceeded");
1740 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1742 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1744 next0 = icmp_out2in_ed_slow_path
1745 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1746 next0, now, thread_index, &s0);
1748 if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP))
1751 reass0->sess_index = s0 - per_thread_data->sessions;
1753 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1754 reass0->thread_index = thread_index;
1755 nat_ip4_reass_get_frags (reass0,
1756 &fragments_to_loopback);
1762 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1763 ip0->protocol, rx_fib_index0, udp0->dst_port,
1766 if (clib_bihash_search_16_8
1767 (&per_thread_data->out2in_ed, &kv0, &value0))
1769 /* Try to match static mapping by external address and port,
1770 destination address and port in packet */
1771 e_key0.addr = ip0->dst_address;
1772 e_key0.port = udp0->dst_port;
1773 e_key0.protocol = proto0;
1774 e_key0.fib_index = rx_fib_index0;
1775 if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1776 &twice_nat0, &lb0, 0,
1780 * Send DHCP packets to the ipv4 stack, or we won't
1781 * be able to use dhcp client on the outside interface
1783 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1786 clib_host_to_net_u16
1787 (UDP_DST_PORT_dhcp_to_client))))
1789 vnet_feature_next (&next0, b0);
1793 if (!sm->forwarding_enabled)
1796 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1797 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1801 if (next_src_nat (sm, ip0, ip0->protocol,
1802 udp0->src_port, udp0->dst_port,
1803 thread_index, rx_fib_index0))
1805 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1808 if (sm->num_workers > 1)
1809 create_bypass_for_fwd_worker (sm, ip0,
1812 create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1814 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1815 nat_ip4_reass_get_frags (reass0,
1816 &fragments_to_loopback);
1821 if (PREDICT_FALSE (identity_nat0))
1823 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1827 if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1829 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1830 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1834 /* Create session initiated by host from external network */
1835 s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1843 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1844 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1847 reass0->sess_index = s0 - per_thread_data->sessions;
1848 reass0->thread_index = thread_index;
1852 s0 = pool_elt_at_index (per_thread_data->sessions,
1854 reass0->sess_index = value0.value;
1856 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1860 if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1862 if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1864 if (nat_ip4_reass_add_fragment
1865 (thread_index, reass0, bi0, &fragments_to_drop))
1867 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_FRAG];
1869 ("maximum fragments per reassembly exceeded");
1870 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1876 s0 = pool_elt_at_index (per_thread_data->sessions,
1877 reass0->sess_index);
1880 old_addr0 = ip0->dst_address.as_u32;
1881 ip0->dst_address = s0->in2out.addr;
1882 new_addr0 = ip0->dst_address.as_u32;
1883 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1885 sum0 = ip0->checksum;
1886 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1888 dst_address /* changed member */ );
1889 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1890 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1891 s0->ext_host_nat_addr.as_u32, ip4_header_t,
1893 ip0->checksum = ip_csum_fold (sum0);
1895 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1897 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1899 old_port0 = tcp0->dst_port;
1900 tcp0->dst_port = s0->in2out.port;
1901 new_port0 = tcp0->dst_port;
1903 sum0 = tcp0->checksum;
1904 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1906 dst_address /* changed member */ );
1908 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1909 ip4_header_t /* cheat */ ,
1910 length /* changed member */ );
1911 if (is_twice_nat_session (s0))
1913 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1914 s0->ext_host_nat_addr.as_u32,
1915 ip4_header_t, dst_address);
1916 sum0 = ip_csum_update (sum0, tcp0->src_port,
1917 s0->ext_host_nat_port,
1918 ip4_header_t, length);
1919 tcp0->src_port = s0->ext_host_nat_port;
1920 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1922 tcp0->checksum = ip_csum_fold (sum0);
1926 old_port0 = udp0->dst_port;
1927 udp0->dst_port = s0->in2out.port;
1928 if (is_twice_nat_session (s0))
1930 udp0->src_port = s0->ext_host_nat_port;
1931 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1938 nat44_session_update_counters (s0, now,
1939 vlib_buffer_length_in_chain (vm, b0),
1941 /* Per-user LRU list maintenance */
1942 nat44_session_update_lru (sm, s0, thread_index);
1945 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1946 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1948 nat44_reass_trace_t *t =
1949 vlib_add_trace (vm, node, b0, sizeof (*t));
1950 t->cached = cached0;
1951 t->sw_if_index = sw_if_index0;
1952 t->next_index = next0;
1962 pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
1964 /* verify speculative enqueue, maybe switch current next frame */
1965 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1966 to_next, n_left_to_next,
1970 if (n_left_from == 0 && vec_len (fragments_to_loopback))
1972 from = vlib_frame_vector_args (frame);
1973 u32 len = vec_len (fragments_to_loopback);
1974 if (len <= VLIB_FRAME_SIZE)
1976 clib_memcpy_fast (from, fragments_to_loopback,
1977 sizeof (u32) * len);
1979 vec_reset_length (fragments_to_loopback);
1983 clib_memcpy_fast (from, fragments_to_loopback +
1984 (len - VLIB_FRAME_SIZE),
1985 sizeof (u32) * VLIB_FRAME_SIZE);
1986 n_left_from = VLIB_FRAME_SIZE;
1987 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1992 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1995 vlib_node_increment_counter (vm, sm->ed_out2in_reass_node_index,
1996 NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1999 nat_send_all_to_node (vm, fragments_to_drop, node,
2000 &node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT],
2001 NAT44_ED_OUT2IN_NEXT_DROP);
2003 vec_free (fragments_to_drop);
2004 vec_free (fragments_to_loopback);
2005 return frame->n_vectors;
2009 VLIB_REGISTER_NODE (nat44_ed_out2in_reass_node) = {
2010 .name = "nat44-ed-out2in-reass",
2011 .vector_size = sizeof (u32),
2012 .format_trace = format_nat44_reass_trace,
2013 .type = VLIB_NODE_TYPE_INTERNAL,
2014 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
2015 .error_strings = nat_out2in_ed_error_strings,
2016 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2018 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2019 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2020 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2021 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2022 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2023 [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
2029 * fd.io coding-style-patch-verification: ON
2032 * eval: (c-set-style "gnu")