2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
17 * @brief NAT44 endpoint-dependent outside to inside network translation
20 #include <vlib/vlib.h>
21 #include <vnet/vnet.h>
22 #include <vnet/pg/pg.h>
23 #include <vnet/ip/ip.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/fib/ip4_fib.h>
26 #include <vnet/udp/udp.h>
27 #include <vppinfra/error.h>
29 #include <nat/nat_ipfix_logging.h>
30 #include <nat/nat_reass.h>
31 #include <nat/nat_inlines.h>
32 #include <nat/nat_syslog.h>
34 #define foreach_nat_out2in_ed_error \
35 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
36 _(OUT2IN_PACKETS, "Good out2in packets processed") \
37 _(OUT_OF_PORTS, "Out of ports") \
38 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
39 _(NO_TRANSLATION, "No translation") \
40 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
41 _(DROP_FRAGMENT, "Drop fragment") \
42 _(MAX_REASS, "Maximum reassemblies exceeded") \
43 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
44 _(NON_SYN, "non-SYN packet try to create session")
48 #define _(sym,str) NAT_OUT2IN_ED_ERROR_##sym,
49 foreach_nat_out2in_ed_error
51 NAT_OUT2IN_ED_N_ERROR,
52 } nat_out2in_ed_error_t;
54 static char *nat_out2in_ed_error_strings[] = {
55 #define _(sym,string) string,
56 foreach_nat_out2in_ed_error
62 NAT44_ED_OUT2IN_NEXT_DROP,
63 NAT44_ED_OUT2IN_NEXT_LOOKUP,
64 NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
65 NAT44_ED_OUT2IN_NEXT_IN2OUT,
66 NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
67 NAT44_ED_OUT2IN_NEXT_REASS,
68 NAT44_ED_OUT2IN_N_NEXT,
69 } nat44_ed_out2in_next_t;
77 } nat44_ed_out2in_trace_t;
79 vlib_node_registration_t nat44_ed_out2in_node;
80 vlib_node_registration_t nat44_ed_out2in_slowpath_node;
81 vlib_node_registration_t nat44_ed_out2in_reass_node;
84 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
86 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
87 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
88 nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
92 t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
93 "NAT44_OUT2IN_ED_FAST_PATH";
95 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
96 t->sw_if_index, t->next_index, t->session_index);
102 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
103 ip4_header_t * ip0, icmp46_header_t * icmp0,
104 u32 sw_if_index0, u32 rx_fib_index0,
105 vlib_node_runtime_t * node, u32 next0, f64 now,
106 u32 thread_index, snat_session_t ** p_s0)
108 next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
109 next0, thread_index, p_s0, 0);
110 snat_session_t *s0 = *p_s0;
111 if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP && s0))
114 nat44_session_update_counters (s0, now,
115 vlib_buffer_length_in_chain
116 (sm->vlib_main, b0));
117 /* Per-user LRU list maintenance */
118 nat44_session_update_lru (sm, s0, thread_index);
124 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
126 snat_main_t *sm = &snat_main;
127 nat44_is_idle_session_ctx_t *ctx = arg;
129 u64 sess_timeout_time;
130 nat_ed_ses_key_t ed_key;
131 clib_bihash_kv_16_8_t ed_kv;
134 snat_session_key_t key;
135 snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
138 s = pool_elt_at_index (tsm->sessions, kv->value);
139 sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
140 if (ctx->now >= sess_timeout_time)
142 ed_key.l_addr = s->in2out.addr;
143 ed_key.r_addr = s->ext_host_addr;
144 ed_key.fib_index = s->in2out.fib_index;
145 if (snat_is_unk_proto_session (s))
147 ed_key.proto = s->in2out.port;
153 ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
154 ed_key.l_port = s->in2out.port;
155 ed_key.r_port = s->ext_host_port;
157 if (is_twice_nat_session (s))
159 ed_key.r_addr = s->ext_host_nat_addr;
160 ed_key.r_port = s->ext_host_nat_port;
162 ed_kv.key[0] = ed_key.as_u64[0];
163 ed_kv.key[1] = ed_key.as_u64[1];
164 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
165 nat_log_warn ("in2out_ed key del failed");
167 if (snat_is_unk_proto_session (s))
170 snat_ipfix_logging_nat44_ses_delete (s->in2out.addr.as_u32,
171 s->out2in.addr.as_u32,
175 s->in2out.fib_index);
177 nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
178 &s->in2out.addr, s->in2out.port,
179 &s->ext_host_nat_addr, s->ext_host_nat_port,
180 &s->out2in.addr, s->out2in.port,
181 &s->ext_host_addr, s->ext_host_port,
182 s->in2out.protocol, is_twice_nat_session (s));
184 if (is_twice_nat_session (s))
186 for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
188 key.protocol = s->in2out.protocol;
189 key.port = s->ext_host_nat_port;
190 a = sm->twice_nat_addresses + i;
191 if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
193 snat_free_outside_address_and_port (sm->twice_nat_addresses,
201 if (snat_is_session_static (s))
204 snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
207 nat44_delete_session (sm, s, ctx->thread_index);
214 static snat_session_t *
215 create_session_for_static_mapping_ed (snat_main_t * sm,
217 snat_session_key_t l_key,
218 snat_session_key_t e_key,
219 vlib_node_runtime_t * node,
221 twice_nat_type_t twice_nat,
222 lb_nat_type_t lb_nat, f64 now)
228 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
229 clib_bihash_kv_16_8_t kv;
230 snat_session_key_t eh_key;
231 nat44_is_idle_session_ctx_t ctx;
233 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
235 b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
236 nat_log_notice ("maximum sessions exceeded");
240 u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
243 nat_log_warn ("create NAT user failed");
247 s = nat_ed_session_alloc (sm, u, thread_index, now);
250 nat44_delete_user_with_no_session (sm, u, thread_index);
251 nat_log_warn ("create NAT session failed");
255 ip = vlib_buffer_get_current (b);
256 udp = ip4_next_header (ip);
258 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
259 s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
260 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
262 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
263 if (lb_nat == AFFINITY_LB_NAT)
264 s->flags |= SNAT_SESSION_FLAG_AFFINITY;
265 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
268 s->in2out.protocol = s->out2in.protocol;
269 user_session_increment (sm, u, 1);
271 /* Add to lookup tables */
272 make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
273 e_key.fib_index, e_key.port, s->ext_host_port);
274 kv.value = s - tsm->sessions;
276 ctx.thread_index = thread_index;
277 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
278 nat44_o2i_ed_is_idle_session_cb,
280 nat_log_notice ("out2in-ed key add failed");
282 if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
283 ip->src_address.as_u32 == l_key.addr.as_u32))
285 eh_key.protocol = e_key.protocol;
286 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
287 thread_index, &eh_key,
289 tsm->snat_thread_index))
291 b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
292 nat44_delete_session (sm, s, thread_index);
293 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
294 nat_log_notice ("out2in-ed key del failed");
297 s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
298 s->ext_host_nat_port = eh_key.port;
299 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
300 make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
301 l_key.fib_index, l_key.port, s->ext_host_nat_port);
305 make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
306 l_key.fib_index, l_key.port, s->ext_host_port);
308 kv.value = s - tsm->sessions;
309 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
310 nat44_i2o_ed_is_idle_session_cb,
312 nat_log_notice ("in2out-ed key add failed");
314 snat_ipfix_logging_nat44_ses_create (s->in2out.addr.as_u32,
315 s->out2in.addr.as_u32,
318 s->out2in.port, s->in2out.fib_index);
320 nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
321 &s->in2out.addr, s->in2out.port,
322 &s->ext_host_nat_addr, s->ext_host_nat_port,
323 &s->out2in.addr, s->out2in.port,
324 &s->ext_host_addr, s->ext_host_port,
325 s->in2out.protocol, is_twice_nat_session (s));
330 static_always_inline int
331 icmp_get_ed_key (ip4_header_t * ip0, nat_ed_ses_key_t * p_key0)
333 icmp46_header_t *icmp0;
334 nat_ed_ses_key_t key0;
335 icmp_echo_header_t *echo0, *inner_echo0 = 0;
336 ip4_header_t *inner_ip0;
338 icmp46_header_t *inner_icmp0;
340 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
341 echo0 = (icmp_echo_header_t *) (icmp0 + 1);
343 if (!icmp_is_error_message (icmp0))
345 key0.proto = IP_PROTOCOL_ICMP;
346 key0.l_addr = ip0->dst_address;
347 key0.r_addr = ip0->src_address;
348 key0.l_port = echo0->identifier;
353 inner_ip0 = (ip4_header_t *) (echo0 + 1);
354 l4_header = ip4_next_header (inner_ip0);
355 key0.proto = inner_ip0->protocol;
356 key0.l_addr = inner_ip0->src_address;
357 key0.r_addr = inner_ip0->dst_address;
358 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
360 case SNAT_PROTOCOL_ICMP:
361 inner_icmp0 = (icmp46_header_t *) l4_header;
362 inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
363 key0.l_port = inner_echo0->identifier;
366 case SNAT_PROTOCOL_UDP:
367 case SNAT_PROTOCOL_TCP:
368 key0.l_port = ((tcp_udp_header_t *) l4_header)->src_port;
369 key0.r_port = ((tcp_udp_header_t *) l4_header)->dst_port;
380 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
381 u16 dst_port, u32 thread_index, u32 rx_fib_index)
383 clib_bihash_kv_16_8_t kv, value;
384 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
386 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
387 rx_fib_index, src_port, dst_port);
388 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
395 create_bypass_for_fwd (snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
398 nat_ed_ses_key_t key;
399 clib_bihash_kv_16_8_t kv, value;
402 snat_session_t *s = 0;
403 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
404 f64 now = vlib_time_now (sm->vlib_main);
406 if (ip->protocol == IP_PROTOCOL_ICMP)
408 if (icmp_get_ed_key (ip, &key))
411 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
413 udp = ip4_next_header (ip);
414 key.r_addr = ip->src_address;
415 key.l_addr = ip->dst_address;
416 key.proto = ip->protocol;
417 key.l_port = udp->dst_port;
418 key.r_port = udp->src_port;
422 key.r_addr = ip->src_address;
423 key.l_addr = ip->dst_address;
424 key.proto = ip->protocol;
425 key.l_port = key.r_port = 0;
428 kv.key[0] = key.as_u64[0];
429 kv.key[1] = key.as_u64[1];
431 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
433 s = pool_elt_at_index (tsm->sessions, value.value);
437 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
440 u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
444 nat_log_warn ("create NAT user failed");
448 s = nat_ed_session_alloc (sm, u, thread_index, now);
451 nat44_delete_user_with_no_session (sm, u, thread_index);
452 nat_log_warn ("create NAT session failed");
456 s->ext_host_addr = key.r_addr;
457 s->ext_host_port = key.r_port;
458 s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
459 s->out2in.addr = key.l_addr;
460 s->out2in.port = key.l_port;
461 s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
462 s->out2in.fib_index = 0;
463 s->in2out = s->out2in;
464 user_session_increment (sm, u, 0);
466 kv.value = s - tsm->sessions;
467 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
468 nat_log_notice ("in2out_ed key add failed");
471 if (ip->protocol == IP_PROTOCOL_TCP)
473 tcp_header_t *tcp = ip4_next_header (ip);
474 if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
479 nat44_session_update_counters (s, now, 0);
480 /* Per-user LRU list maintenance */
481 nat44_session_update_lru (sm, s, thread_index);
485 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
486 u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
487 u8 * p_proto, snat_session_key_t * p_value,
488 u8 * p_dont_translate, void *d, void *e)
490 u32 next = ~0, sw_if_index, rx_fib_index;
491 icmp46_header_t *icmp;
492 nat_ed_ses_key_t key;
493 clib_bihash_kv_16_8_t kv, value;
494 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
495 snat_session_t *s = 0;
496 u8 dont_translate = 0, is_addr_only, identity_nat;
497 snat_session_key_t e_key, l_key;
499 icmp = (icmp46_header_t *) ip4_next_header (ip);
500 sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
501 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
503 if (icmp_get_ed_key (ip, &key))
505 b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
506 next = NAT44_ED_OUT2IN_NEXT_DROP;
509 key.fib_index = rx_fib_index;
510 kv.key[0] = key.as_u64[0];
511 kv.key[1] = key.as_u64[1];
513 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
515 /* Try to match static mapping */
516 e_key.addr = ip->dst_address;
517 e_key.port = key.l_port;
518 e_key.protocol = ip_proto_to_snat_proto (key.proto);
519 e_key.fib_index = rx_fib_index;
520 if (snat_static_mapping_match
521 (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0, &identity_nat))
523 if (!sm->forwarding_enabled)
525 /* Don't NAT packet aimed at the intfc address */
526 if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
527 ip->dst_address.as_u32)))
532 b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
533 next = NAT44_ED_OUT2IN_NEXT_DROP;
539 if (next_src_nat (sm, ip, key.proto, key.l_port, key.r_port,
540 thread_index, rx_fib_index))
542 next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
545 create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
550 if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
551 (icmp->type != ICMP4_echo_request || !is_addr_only)))
553 b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
554 next = NAT44_ED_OUT2IN_NEXT_DROP;
558 if (PREDICT_FALSE (identity_nat))
564 /* Create session initiated by host from external network */
565 s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
572 next = NAT44_ED_OUT2IN_NEXT_DROP;
578 if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
579 icmp->type != ICMP4_echo_request &&
580 !icmp_is_error_message (icmp)))
582 b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
583 next = NAT44_ED_OUT2IN_NEXT_DROP;
587 s = pool_elt_at_index (tsm->sessions, value.value);
590 *p_proto = ip_proto_to_snat_proto (key.proto);
593 *p_value = s->in2out;
594 *p_dont_translate = dont_translate;
596 *(snat_session_t **) d = s;
600 static snat_session_t *
601 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
607 vlib_main_t * vm, vlib_node_runtime_t * node)
609 clib_bihash_kv_8_8_t kv, value;
610 clib_bihash_kv_16_8_t s_kv, s_value;
611 snat_static_mapping_t *m;
612 u32 old_addr, new_addr;
615 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
618 old_addr = ip->dst_address.as_u32;
620 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
623 if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
625 s = pool_elt_at_index (tsm->sessions, s_value.value);
626 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
630 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
632 b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
633 nat_log_notice ("maximum sessions exceeded");
637 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
638 if (clib_bihash_search_8_8
639 (&sm->static_mapping_by_external, &kv, &value))
641 b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
645 m = pool_elt_at_index (sm->static_mappings, value.value);
647 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
649 u = nat_user_get_or_create (sm, &m->local_addr, m->fib_index,
653 nat_log_warn ("create NAT user failed");
657 /* Create a new session */
658 s = nat_ed_session_alloc (sm, u, thread_index, now);
661 nat44_delete_user_with_no_session (sm, u, thread_index);
662 nat_log_warn ("create NAT session failed");
666 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
667 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
668 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
669 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
670 s->out2in.addr.as_u32 = old_addr;
671 s->out2in.fib_index = rx_fib_index;
672 s->in2out.addr.as_u32 = new_addr;
673 s->in2out.fib_index = m->fib_index;
674 s->in2out.port = s->out2in.port = ip->protocol;
675 user_session_increment (sm, u, 1);
677 /* Add to lookup tables */
678 s_kv.value = s - tsm->sessions;
679 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
680 nat_log_notice ("out2in key add failed");
682 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
684 s_kv.value = s - tsm->sessions;
685 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
686 nat_log_notice ("in2out key add failed");
689 /* Update IP checksum */
691 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
692 ip->checksum = ip_csum_fold (sum);
694 vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
697 nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
698 /* Per-user LRU list maintenance */
699 nat44_session_update_lru (sm, s, thread_index);
705 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
706 vlib_node_runtime_t * node,
707 vlib_frame_t * frame, int is_slow_path)
709 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
710 nat44_ed_out2in_next_t next_index;
711 snat_main_t *sm = &snat_main;
712 f64 now = vlib_time_now (vm);
713 u32 thread_index = vm->thread_index;
714 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
716 stats_node_index = is_slow_path ? nat44_ed_out2in_slowpath_node.index :
717 nat44_ed_out2in_node.index;
719 from = vlib_frame_vector_args (frame);
720 n_left_from = frame->n_vectors;
721 next_index = node->cached_next_index;
723 while (n_left_from > 0)
727 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
729 while (n_left_from >= 4 && n_left_to_next >= 2)
732 vlib_buffer_t *b0, *b1;
733 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
735 u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1,
737 u16 old_port0, new_port0, old_port1, new_port1;
738 ip4_header_t *ip0, *ip1;
739 udp_header_t *udp0, *udp1;
740 tcp_header_t *tcp0, *tcp1;
741 icmp46_header_t *icmp0, *icmp1;
742 snat_session_t *s0 = 0, *s1 = 0;
743 clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
744 ip_csum_t sum0, sum1;
745 snat_session_key_t e_key0, l_key0, e_key1, l_key1;
746 lb_nat_type_t lb_nat0, lb_nat1;
747 twice_nat_type_t twice_nat0, twice_nat1;
748 u8 identity_nat0, identity_nat1;
750 /* Prefetch next iteration. */
752 vlib_buffer_t *p2, *p3;
754 p2 = vlib_get_buffer (vm, from[2]);
755 p3 = vlib_get_buffer (vm, from[3]);
757 vlib_prefetch_buffer_header (p2, LOAD);
758 vlib_prefetch_buffer_header (p3, LOAD);
760 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
761 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
764 /* speculatively enqueue b0 and b1 to the current next frame */
765 to_next[0] = bi0 = from[0];
766 to_next[1] = bi1 = from[1];
772 b0 = vlib_get_buffer (vm, bi0);
773 b1 = vlib_get_buffer (vm, bi1);
775 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
776 vnet_buffer (b0)->snat.flags = 0;
777 ip0 = vlib_buffer_get_current (b0);
779 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
781 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
784 if (PREDICT_FALSE (ip0->ttl == 1))
786 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
787 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
788 ICMP4_time_exceeded_ttl_exceeded_in_transit,
790 next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
794 udp0 = ip4_next_header (ip0);
795 tcp0 = (tcp_header_t *) udp0;
796 icmp0 = (icmp46_header_t *) udp0;
797 proto0 = ip_proto_to_snat_proto (ip0->protocol);
801 if (PREDICT_FALSE (proto0 == ~0))
804 nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
805 thread_index, now, vm,
807 if (!sm->forwarding_enabled)
810 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
815 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
817 next0 = icmp_out2in_ed_slow_path
818 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
819 next0, now, thread_index, &s0);
825 if (PREDICT_FALSE (proto0 == ~0))
827 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
831 if (ip4_is_fragment (ip0))
833 next0 = NAT44_ED_OUT2IN_NEXT_REASS;
837 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
839 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
844 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
845 ip0->protocol, rx_fib_index0, udp0->dst_port,
848 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
852 /* Try to match static mapping by external address and port,
853 destination address and port in packet */
854 e_key0.addr = ip0->dst_address;
855 e_key0.port = udp0->dst_port;
856 e_key0.protocol = proto0;
857 e_key0.fib_index = rx_fib_index0;
858 if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
859 &twice_nat0, &lb_nat0,
864 * Send DHCP packets to the ipv4 stack, or we won't
865 * be able to use dhcp client on the outside interface
867 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
868 && (udp0->dst_port ==
870 (UDP_DST_PORT_dhcp_to_client))))
872 vnet_feature_next (&next0, b0);
876 if (!sm->forwarding_enabled)
879 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
880 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
884 if (next_src_nat (sm, ip0, ip0->protocol,
885 udp0->src_port, udp0->dst_port,
886 thread_index, rx_fib_index0))
888 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
891 create_bypass_for_fwd (sm, ip0, rx_fib_index0,
897 if (PREDICT_FALSE (identity_nat0))
900 if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
902 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
903 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
907 /* Create session initiated by host from external network */
908 s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
916 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
922 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
928 s0 = pool_elt_at_index (tsm->sessions, value0.value);
931 old_addr0 = ip0->dst_address.as_u32;
932 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
933 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
935 sum0 = ip0->checksum;
936 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
938 if (PREDICT_FALSE (is_twice_nat_session (s0)))
939 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
940 s0->ext_host_nat_addr.as_u32, ip4_header_t,
942 ip0->checksum = ip_csum_fold (sum0);
944 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
946 old_port0 = tcp0->dst_port;
947 new_port0 = tcp0->dst_port = s0->in2out.port;
949 sum0 = tcp0->checksum;
950 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
952 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
954 if (is_twice_nat_session (s0))
956 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
957 s0->ext_host_nat_addr.as_u32,
958 ip4_header_t, dst_address);
959 sum0 = ip_csum_update (sum0, tcp0->src_port,
960 s0->ext_host_nat_port, ip4_header_t,
962 tcp0->src_port = s0->ext_host_nat_port;
963 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
965 tcp0->checksum = ip_csum_fold (sum0);
966 if (nat44_set_tcp_session_state_o2i
967 (sm, s0, tcp0, thread_index))
972 udp0->dst_port = s0->in2out.port;
973 if (is_twice_nat_session (s0))
975 udp0->src_port = s0->ext_host_nat_port;
976 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
982 nat44_session_update_counters (s0, now,
983 vlib_buffer_length_in_chain (vm,
985 /* Per-user LRU list maintenance */
986 nat44_session_update_lru (sm, s0, thread_index);
989 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
990 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
992 nat44_ed_out2in_trace_t *t =
993 vlib_add_trace (vm, node, b0, sizeof (*t));
994 t->is_slow_path = is_slow_path;
995 t->sw_if_index = sw_if_index0;
996 t->next_index = next0;
997 t->session_index = ~0;
999 t->session_index = s0 - tsm->sessions;
1002 pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
1004 next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1005 vnet_buffer (b1)->snat.flags = 0;
1006 ip1 = vlib_buffer_get_current (b1);
1008 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1010 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1013 if (PREDICT_FALSE (ip1->ttl == 1))
1015 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1016 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1017 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1019 next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1023 udp1 = ip4_next_header (ip1);
1024 tcp1 = (tcp_header_t *) udp1;
1025 icmp1 = (icmp46_header_t *) udp1;
1026 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1030 if (PREDICT_FALSE (proto1 == ~0))
1033 nat44_ed_out2in_unknown_proto (sm, b1, ip1, rx_fib_index1,
1034 thread_index, now, vm,
1036 if (!sm->forwarding_enabled)
1039 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1044 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1046 next1 = icmp_out2in_ed_slow_path
1047 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1048 next1, now, thread_index, &s1);
1054 if (PREDICT_FALSE (proto1 == ~0))
1056 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1060 if (ip4_is_fragment (ip1))
1062 next1 = NAT44_ED_OUT2IN_NEXT_REASS;
1066 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1068 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1073 make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address,
1074 ip1->protocol, rx_fib_index1, udp1->dst_port,
1077 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
1081 /* Try to match static mapping by external address and port,
1082 destination address and port in packet */
1083 e_key1.addr = ip1->dst_address;
1084 e_key1.port = udp1->dst_port;
1085 e_key1.protocol = proto1;
1086 e_key1.fib_index = rx_fib_index1;
1087 if (snat_static_mapping_match (sm, e_key1, &l_key1, 1, 0,
1088 &twice_nat1, &lb_nat1,
1093 * Send DHCP packets to the ipv4 stack, or we won't
1094 * be able to use dhcp client on the outside interface
1096 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1097 && (udp1->dst_port ==
1098 clib_host_to_net_u16
1099 (UDP_DST_PORT_dhcp_to_client))))
1101 vnet_feature_next (&next1, b1);
1105 if (!sm->forwarding_enabled)
1108 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1109 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1113 if (next_src_nat (sm, ip1, ip1->protocol,
1114 udp1->src_port, udp1->dst_port,
1115 thread_index, rx_fib_index1))
1117 next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1120 create_bypass_for_fwd (sm, ip1, rx_fib_index1,
1126 if (PREDICT_FALSE (identity_nat1))
1129 if ((proto1 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp1))
1131 b1->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1132 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1136 /* Create session initiated by host from external network */
1137 s1 = create_session_for_static_mapping_ed (sm, b1, l_key1,
1145 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
1151 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1157 s1 = pool_elt_at_index (tsm->sessions, value1.value);
1160 old_addr1 = ip1->dst_address.as_u32;
1161 new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
1162 vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1164 sum1 = ip1->checksum;
1165 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1167 if (PREDICT_FALSE (is_twice_nat_session (s1)))
1168 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1169 s1->ext_host_nat_addr.as_u32, ip4_header_t,
1171 ip1->checksum = ip_csum_fold (sum1);
1173 if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1175 old_port1 = tcp1->dst_port;
1176 new_port1 = tcp1->dst_port = s1->in2out.port;
1178 sum1 = tcp1->checksum;
1179 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1181 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1183 if (is_twice_nat_session (s1))
1185 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1186 s1->ext_host_nat_addr.as_u32,
1187 ip4_header_t, dst_address);
1188 sum1 = ip_csum_update (sum1, tcp1->src_port,
1189 s1->ext_host_nat_port, ip4_header_t,
1191 tcp1->src_port = s1->ext_host_nat_port;
1192 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1194 tcp1->checksum = ip_csum_fold (sum1);
1195 if (nat44_set_tcp_session_state_o2i
1196 (sm, s1, tcp1, thread_index))
1201 udp1->dst_port = s1->in2out.port;
1202 if (is_twice_nat_session (s1))
1204 udp1->src_port = s1->ext_host_nat_port;
1205 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1211 nat44_session_update_counters (s1, now,
1212 vlib_buffer_length_in_chain (vm,
1214 /* Per-user LRU list maintenance */
1215 nat44_session_update_lru (sm, s1, thread_index);
1218 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1219 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1221 nat44_ed_out2in_trace_t *t =
1222 vlib_add_trace (vm, node, b1, sizeof (*t));
1223 t->is_slow_path = is_slow_path;
1224 t->sw_if_index = sw_if_index1;
1225 t->next_index = next1;
1226 t->session_index = ~0;
1228 t->session_index = s1 - tsm->sessions;
1231 pkts_processed += next1 != NAT44_ED_OUT2IN_NEXT_DROP;
1233 /* verify speculative enqueues, maybe switch current next frame */
1234 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1235 to_next, n_left_to_next,
1236 bi0, bi1, next0, next1);
1239 while (n_left_from > 0 && n_left_to_next > 0)
1243 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
1245 u16 old_port0, new_port0;
1249 icmp46_header_t *icmp0;
1250 snat_session_t *s0 = 0;
1251 clib_bihash_kv_16_8_t kv0, value0;
1253 snat_session_key_t e_key0, l_key0;
1254 lb_nat_type_t lb_nat0;
1255 twice_nat_type_t twice_nat0;
1258 /* speculatively enqueue b0 to the current next frame */
1264 n_left_to_next -= 1;
1266 b0 = vlib_get_buffer (vm, bi0);
1267 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1268 vnet_buffer (b0)->snat.flags = 0;
1269 ip0 = vlib_buffer_get_current (b0);
1271 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1273 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1276 if (PREDICT_FALSE (ip0->ttl == 1))
1278 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1279 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1280 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1282 next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
1286 udp0 = ip4_next_header (ip0);
1287 tcp0 = (tcp_header_t *) udp0;
1288 icmp0 = (icmp46_header_t *) udp0;
1289 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1293 if (PREDICT_FALSE (proto0 == ~0))
1296 nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1297 thread_index, now, vm,
1299 if (!sm->forwarding_enabled)
1302 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1307 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1309 next0 = icmp_out2in_ed_slow_path
1310 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1311 next0, now, thread_index, &s0);
1317 if (PREDICT_FALSE (proto0 == ~0))
1319 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1323 if (ip4_is_fragment (ip0))
1325 next0 = NAT44_ED_OUT2IN_NEXT_REASS;
1329 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1331 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1336 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1337 ip0->protocol, rx_fib_index0, udp0->dst_port,
1340 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
1344 /* Try to match static mapping by external address and port,
1345 destination address and port in packet */
1346 e_key0.addr = ip0->dst_address;
1347 e_key0.port = udp0->dst_port;
1348 e_key0.protocol = proto0;
1349 e_key0.fib_index = rx_fib_index0;
1350 if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1351 &twice_nat0, &lb_nat0,
1356 * Send DHCP packets to the ipv4 stack, or we won't
1357 * be able to use dhcp client on the outside interface
1359 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1360 && (udp0->dst_port ==
1361 clib_host_to_net_u16
1362 (UDP_DST_PORT_dhcp_to_client))))
1364 vnet_feature_next (&next0, b0);
1368 if (!sm->forwarding_enabled)
1371 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1372 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1376 if (next_src_nat (sm, ip0, ip0->protocol,
1377 udp0->src_port, udp0->dst_port,
1378 thread_index, rx_fib_index0))
1380 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1383 create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1389 if (PREDICT_FALSE (identity_nat0))
1392 if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1394 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1395 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1399 /* Create session initiated by host from external network */
1400 s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1408 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1414 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
1420 s0 = pool_elt_at_index (tsm->sessions, value0.value);
1423 old_addr0 = ip0->dst_address.as_u32;
1424 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1425 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1427 sum0 = ip0->checksum;
1428 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1430 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1431 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1432 s0->ext_host_nat_addr.as_u32, ip4_header_t,
1434 ip0->checksum = ip_csum_fold (sum0);
1436 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1438 old_port0 = tcp0->dst_port;
1439 new_port0 = tcp0->dst_port = s0->in2out.port;
1441 sum0 = tcp0->checksum;
1442 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1444 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1446 if (is_twice_nat_session (s0))
1448 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1449 s0->ext_host_nat_addr.as_u32,
1450 ip4_header_t, dst_address);
1451 sum0 = ip_csum_update (sum0, tcp0->src_port,
1452 s0->ext_host_nat_port, ip4_header_t,
1454 tcp0->src_port = s0->ext_host_nat_port;
1455 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1457 tcp0->checksum = ip_csum_fold (sum0);
1458 if (nat44_set_tcp_session_state_o2i
1459 (sm, s0, tcp0, thread_index))
1464 udp0->dst_port = s0->in2out.port;
1465 if (is_twice_nat_session (s0))
1467 udp0->src_port = s0->ext_host_nat_port;
1468 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1474 nat44_session_update_counters (s0, now,
1475 vlib_buffer_length_in_chain (vm,
1477 /* Per-user LRU list maintenance */
1478 nat44_session_update_lru (sm, s0, thread_index);
1481 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1482 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1484 nat44_ed_out2in_trace_t *t =
1485 vlib_add_trace (vm, node, b0, sizeof (*t));
1486 t->is_slow_path = is_slow_path;
1487 t->sw_if_index = sw_if_index0;
1488 t->next_index = next0;
1489 t->session_index = ~0;
1491 t->session_index = s0 - tsm->sessions;
1494 pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
1495 /* verify speculative enqueue, maybe switch current next frame */
1496 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1497 to_next, n_left_to_next,
1501 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1504 vlib_node_increment_counter (vm, stats_node_index,
1505 NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1507 return frame->n_vectors;
1511 nat44_ed_out2in_fast_path_fn (vlib_main_t * vm,
1512 vlib_node_runtime_t * node,
1513 vlib_frame_t * frame)
1515 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
1519 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1520 .function = nat44_ed_out2in_fast_path_fn,
1521 .name = "nat44-ed-out2in",
1522 .vector_size = sizeof (u32),
1523 .format_trace = format_nat44_ed_out2in_trace,
1524 .type = VLIB_NODE_TYPE_INTERNAL,
1525 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1526 .error_strings = nat_out2in_ed_error_strings,
1527 .runtime_data_bytes = sizeof (snat_runtime_t),
1528 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1530 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1531 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1532 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1533 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1534 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1535 [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1540 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_node,
1541 nat44_ed_out2in_fast_path_fn);
1544 nat44_ed_out2in_slow_path_fn (vlib_main_t * vm,
1545 vlib_node_runtime_t * node,
1546 vlib_frame_t * frame)
1548 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
1552 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1553 .function = nat44_ed_out2in_slow_path_fn,
1554 .name = "nat44-ed-out2in-slowpath",
1555 .vector_size = sizeof (u32),
1556 .format_trace = format_nat44_ed_out2in_trace,
1557 .type = VLIB_NODE_TYPE_INTERNAL,
1558 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1559 .error_strings = nat_out2in_ed_error_strings,
1560 .runtime_data_bytes = sizeof (snat_runtime_t),
1561 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1563 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1564 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1565 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1566 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1567 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1568 [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1573 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_slowpath_node,
1574 nat44_ed_out2in_slow_path_fn);
1577 nat44_ed_out2in_reass_node_fn (vlib_main_t * vm,
1578 vlib_node_runtime_t * node,
1579 vlib_frame_t * frame)
1581 u32 n_left_from, *from, *to_next;
1582 nat44_ed_out2in_next_t next_index;
1583 u32 pkts_processed = 0;
1584 snat_main_t *sm = &snat_main;
1585 f64 now = vlib_time_now (vm);
1586 u32 thread_index = vm->thread_index;
1587 snat_main_per_thread_data_t *per_thread_data =
1588 &sm->per_thread_data[thread_index];
1589 u32 *fragments_to_drop = 0;
1590 u32 *fragments_to_loopback = 0;
1592 from = vlib_frame_vector_args (frame);
1593 n_left_from = frame->n_vectors;
1594 next_index = node->cached_next_index;
1596 while (n_left_from > 0)
1600 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1602 while (n_left_from > 0 && n_left_to_next > 0)
1604 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1609 nat_reass_ip4_t *reass0;
1612 icmp46_header_t *icmp0;
1613 clib_bihash_kv_16_8_t kv0, value0;
1614 snat_session_t *s0 = 0;
1615 u16 old_port0, new_port0;
1617 snat_session_key_t e_key0, l_key0;
1619 twice_nat_type_t twice_nat0;
1622 /* speculatively enqueue b0 to the current next frame */
1628 n_left_to_next -= 1;
1630 b0 = vlib_get_buffer (vm, bi0);
1631 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
1633 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1635 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1638 if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1640 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1641 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT];
1645 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1646 udp0 = ip4_next_header (ip0);
1647 tcp0 = (tcp_header_t *) udp0;
1648 icmp0 = (icmp46_header_t *) udp0;
1649 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1651 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1655 1, &fragments_to_drop);
1657 if (PREDICT_FALSE (!reass0))
1659 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1660 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_REASS];
1661 nat_log_notice ("maximum reassemblies exceeded");
1665 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1667 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1669 next0 = icmp_out2in_ed_slow_path
1670 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1671 next0, now, thread_index, &s0);
1673 if (PREDICT_TRUE (next0 != NAT44_ED_OUT2IN_NEXT_DROP))
1676 reass0->sess_index = s0 - per_thread_data->sessions;
1678 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1679 reass0->thread_index = thread_index;
1680 nat_ip4_reass_get_frags (reass0,
1681 &fragments_to_loopback);
1687 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1688 ip0->protocol, rx_fib_index0, udp0->dst_port,
1691 if (clib_bihash_search_16_8
1692 (&per_thread_data->out2in_ed, &kv0, &value0))
1694 /* Try to match static mapping by external address and port,
1695 destination address and port in packet */
1696 e_key0.addr = ip0->dst_address;
1697 e_key0.port = udp0->dst_port;
1698 e_key0.protocol = proto0;
1699 e_key0.fib_index = rx_fib_index0;
1700 if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1701 &twice_nat0, &lb0, 0,
1705 * Send DHCP packets to the ipv4 stack, or we won't
1706 * be able to use dhcp client on the outside interface
1708 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1711 clib_host_to_net_u16
1712 (UDP_DST_PORT_dhcp_to_client))))
1714 vnet_feature_next (&next0, b0);
1718 if (!sm->forwarding_enabled)
1721 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1722 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1726 if (next_src_nat (sm, ip0, ip0->protocol,
1727 udp0->src_port, udp0->dst_port,
1728 thread_index, rx_fib_index0))
1730 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1733 create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1735 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1736 nat_ip4_reass_get_frags (reass0,
1737 &fragments_to_loopback);
1742 if (PREDICT_FALSE (identity_nat0))
1744 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1748 if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1750 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1751 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1755 /* Create session initiated by host from external network */
1756 s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1764 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1765 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1768 reass0->sess_index = s0 - per_thread_data->sessions;
1769 reass0->thread_index = thread_index;
1773 s0 = pool_elt_at_index (per_thread_data->sessions,
1775 reass0->sess_index = value0.value;
1777 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1781 if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1783 if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1785 if (nat_ip4_reass_add_fragment
1786 (reass0, bi0, &fragments_to_drop))
1788 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_FRAG];
1790 ("maximum fragments per reassembly exceeded");
1791 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
1797 s0 = pool_elt_at_index (per_thread_data->sessions,
1798 reass0->sess_index);
1801 old_addr0 = ip0->dst_address.as_u32;
1802 ip0->dst_address = s0->in2out.addr;
1803 new_addr0 = ip0->dst_address.as_u32;
1804 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1806 sum0 = ip0->checksum;
1807 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1809 dst_address /* changed member */ );
1810 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1811 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1812 s0->ext_host_nat_addr.as_u32, ip4_header_t,
1814 ip0->checksum = ip_csum_fold (sum0);
1816 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1818 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1820 old_port0 = tcp0->dst_port;
1821 tcp0->dst_port = s0->in2out.port;
1822 new_port0 = tcp0->dst_port;
1824 sum0 = tcp0->checksum;
1825 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1827 dst_address /* changed member */ );
1829 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1830 ip4_header_t /* cheat */ ,
1831 length /* changed member */ );
1832 if (is_twice_nat_session (s0))
1834 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1835 s0->ext_host_nat_addr.as_u32,
1836 ip4_header_t, dst_address);
1837 sum0 = ip_csum_update (sum0, tcp0->src_port,
1838 s0->ext_host_nat_port,
1839 ip4_header_t, length);
1840 tcp0->src_port = s0->ext_host_nat_port;
1841 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1843 tcp0->checksum = ip_csum_fold (sum0);
1847 old_port0 = udp0->dst_port;
1848 udp0->dst_port = s0->in2out.port;
1849 if (is_twice_nat_session (s0))
1851 udp0->src_port = s0->ext_host_nat_port;
1852 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1859 nat44_session_update_counters (s0, now,
1860 vlib_buffer_length_in_chain (vm,
1862 /* Per-user LRU list maintenance */
1863 nat44_session_update_lru (sm, s0, thread_index);
1866 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1867 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1869 nat44_reass_trace_t *t =
1870 vlib_add_trace (vm, node, b0, sizeof (*t));
1871 t->cached = cached0;
1872 t->sw_if_index = sw_if_index0;
1873 t->next_index = next0;
1883 pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
1885 /* verify speculative enqueue, maybe switch current next frame */
1886 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1887 to_next, n_left_to_next,
1891 if (n_left_from == 0 && vec_len (fragments_to_loopback))
1893 from = vlib_frame_vector_args (frame);
1894 u32 len = vec_len (fragments_to_loopback);
1895 if (len <= VLIB_FRAME_SIZE)
1897 clib_memcpy_fast (from, fragments_to_loopback,
1898 sizeof (u32) * len);
1900 vec_reset_length (fragments_to_loopback);
1904 clib_memcpy_fast (from, fragments_to_loopback +
1905 (len - VLIB_FRAME_SIZE),
1906 sizeof (u32) * VLIB_FRAME_SIZE);
1907 n_left_from = VLIB_FRAME_SIZE;
1908 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1913 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1916 vlib_node_increment_counter (vm, nat44_ed_out2in_reass_node.index,
1917 NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1920 nat_send_all_to_node (vm, fragments_to_drop, node,
1921 &node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT],
1922 NAT44_ED_OUT2IN_NEXT_DROP);
1924 vec_free (fragments_to_drop);
1925 vec_free (fragments_to_loopback);
1926 return frame->n_vectors;
1930 VLIB_REGISTER_NODE (nat44_ed_out2in_reass_node) = {
1931 .function = nat44_ed_out2in_reass_node_fn,
1932 .name = "nat44-ed-out2in-reass",
1933 .vector_size = sizeof (u32),
1934 .format_trace = format_nat44_reass_trace,
1935 .type = VLIB_NODE_TYPE_INTERNAL,
1936 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1937 .error_strings = nat_out2in_ed_error_strings,
1938 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
1940 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
1941 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1942 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
1943 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1944 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
1945 [NAT44_ED_OUT2IN_NEXT_REASS] = "nat44-ed-out2in-reass",
1950 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_reass_node,
1951 nat44_ed_out2in_reass_node_fn);
1954 * fd.io coding-style-patch-verification: ON
1957 * eval: (c-set-style "gnu")