2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
17 * @brief NAT44 endpoint-dependent outside to inside network translation
20 #include <vlib/vlib.h>
21 #include <vnet/vnet.h>
22 #include <vnet/pg/pg.h>
23 #include <vnet/ip/ip.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/fib/ip4_fib.h>
26 #include <vnet/udp/udp.h>
27 #include <vppinfra/error.h>
29 #include <nat/nat_ipfix_logging.h>
30 #include <nat/nat_reass.h>
31 #include <nat/nat_inlines.h>
32 #include <nat/nat44_inlines.h>
33 #include <nat/nat_syslog.h>
34 #include <nat/nat_ha.h>
36 static char *nat_out2in_ed_error_strings[] = {
37 #define _(sym,string) string,
38 foreach_nat_out2in_ed_error
48 } nat44_ed_out2in_trace_t;
51 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
53 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
54 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
55 nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
59 t->is_slow_path ? "NAT44_OUT2IN_ED_SLOW_PATH" :
60 "NAT44_OUT2IN_ED_FAST_PATH";
62 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
63 t->sw_if_index, t->next_index, t->session_index);
69 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
70 ip4_header_t * ip0, icmp46_header_t * icmp0,
71 u32 sw_if_index0, u32 rx_fib_index0,
72 vlib_node_runtime_t * node, u32 next0, f64 now,
73 u32 thread_index, snat_session_t ** p_s0)
75 next0 = icmp_out2in (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
76 next0, thread_index, p_s0, 0);
77 snat_session_t *s0 = *p_s0;
78 if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
81 nat44_session_update_counters (s0, now,
82 vlib_buffer_length_in_chain
83 (sm->vlib_main, b0), thread_index);
84 /* Per-user LRU list maintenance */
85 nat44_session_update_lru (sm, s0, thread_index);
90 #ifndef CLIB_MARCH_VARIANT
92 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
94 snat_main_t *sm = &snat_main;
95 nat44_is_idle_session_ctx_t *ctx = arg;
97 u64 sess_timeout_time;
98 nat_ed_ses_key_t ed_key;
99 clib_bihash_kv_16_8_t ed_kv;
102 snat_session_key_t key;
103 snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
106 s = pool_elt_at_index (tsm->sessions, kv->value);
107 sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
108 if (ctx->now >= sess_timeout_time)
110 ed_key.l_addr = s->in2out.addr;
111 ed_key.r_addr = s->ext_host_addr;
112 ed_key.fib_index = s->in2out.fib_index;
113 if (snat_is_unk_proto_session (s))
115 ed_key.proto = s->in2out.port;
121 ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
122 ed_key.l_port = s->in2out.port;
123 ed_key.r_port = s->ext_host_port;
125 if (is_twice_nat_session (s))
127 ed_key.r_addr = s->ext_host_nat_addr;
128 ed_key.r_port = s->ext_host_nat_port;
130 ed_kv.key[0] = ed_key.as_u64[0];
131 ed_kv.key[1] = ed_key.as_u64[1];
132 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
133 nat_elog_warn ("in2out_ed key del failed");
135 if (snat_is_unk_proto_session (s))
138 snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
139 s->in2out.addr.as_u32,
140 s->out2in.addr.as_u32,
144 s->in2out.fib_index);
146 nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
147 &s->in2out.addr, s->in2out.port,
148 &s->ext_host_nat_addr, s->ext_host_nat_port,
149 &s->out2in.addr, s->out2in.port,
150 &s->ext_host_addr, s->ext_host_port,
151 s->in2out.protocol, is_twice_nat_session (s));
153 nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
154 s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
157 if (is_twice_nat_session (s))
159 for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
161 key.protocol = s->in2out.protocol;
162 key.port = s->ext_host_nat_port;
163 a = sm->twice_nat_addresses + i;
164 if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
166 snat_free_outside_address_and_port (sm->twice_nat_addresses,
174 if (snat_is_session_static (s))
177 snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
180 nat44_delete_session (sm, s, ctx->thread_index);
188 static snat_session_t *
189 create_session_for_static_mapping_ed (snat_main_t * sm,
191 snat_session_key_t l_key,
192 snat_session_key_t e_key,
193 vlib_node_runtime_t * node,
195 twice_nat_type_t twice_nat,
196 lb_nat_type_t lb_nat, f64 now)
202 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
203 clib_bihash_kv_16_8_t kv;
204 snat_session_key_t eh_key;
205 nat44_is_idle_session_ctx_t ctx;
207 nat44_session_try_cleanup (&l_key.addr, l_key.fib_index, thread_index, now);
209 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
211 b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
212 nat_elog_notice ("maximum sessions exceeded");
216 u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
219 nat_elog_warn ("create NAT user failed");
223 s = nat_ed_session_alloc (sm, u, thread_index, now);
226 nat44_delete_user_with_no_session (sm, u, thread_index);
227 nat_elog_warn ("create NAT session failed");
231 ip = vlib_buffer_get_current (b);
232 udp = ip4_next_header (ip);
234 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
235 s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
236 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
238 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
239 if (lb_nat == AFFINITY_LB_NAT)
240 s->flags |= SNAT_SESSION_FLAG_AFFINITY;
241 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
244 s->in2out.protocol = s->out2in.protocol;
245 user_session_increment (sm, u, 1);
247 /* Add to lookup tables */
248 make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
249 e_key.fib_index, e_key.port, s->ext_host_port);
250 kv.value = s - tsm->sessions;
252 ctx.thread_index = thread_index;
253 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
254 nat44_o2i_ed_is_idle_session_cb,
256 nat_elog_notice ("out2in-ed key add failed");
258 if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
259 ip->src_address.as_u32 == l_key.addr.as_u32))
261 eh_key.protocol = e_key.protocol;
262 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
263 thread_index, &eh_key,
265 tsm->snat_thread_index))
267 b->error = node->errors[NAT_OUT2IN_ED_ERROR_OUT_OF_PORTS];
268 nat44_delete_session (sm, s, thread_index);
269 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
270 nat_elog_notice ("out2in-ed key del failed");
273 s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
274 s->ext_host_nat_port = eh_key.port;
275 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
276 make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
277 l_key.fib_index, l_key.port, s->ext_host_nat_port);
281 make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
282 l_key.fib_index, l_key.port, s->ext_host_port);
284 kv.value = s - tsm->sessions;
285 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
286 nat44_i2o_ed_is_idle_session_cb,
288 nat_elog_notice ("in2out-ed key add failed");
290 snat_ipfix_logging_nat44_ses_create (thread_index,
291 s->in2out.addr.as_u32,
292 s->out2in.addr.as_u32,
295 s->out2in.port, s->in2out.fib_index);
297 nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
298 &s->in2out.addr, s->in2out.port,
299 &s->ext_host_nat_addr, s->ext_host_nat_port,
300 &s->out2in.addr, s->out2in.port,
301 &s->ext_host_addr, s->ext_host_port,
302 s->in2out.protocol, is_twice_nat_session (s));
304 nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
305 s->out2in.port, &s->ext_host_addr, s->ext_host_port,
306 &s->ext_host_nat_addr, s->ext_host_nat_port,
307 s->in2out.protocol, s->in2out.fib_index, s->flags,
314 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
315 u16 dst_port, u32 thread_index, u32 rx_fib_index)
317 clib_bihash_kv_16_8_t kv, value;
318 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
320 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
321 rx_fib_index, src_port, dst_port);
322 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
329 create_bypass_for_fwd (snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
332 nat_ed_ses_key_t key;
333 clib_bihash_kv_16_8_t kv, value;
336 snat_session_t *s = 0;
337 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
338 f64 now = vlib_time_now (sm->vlib_main);
340 if (ip->protocol == IP_PROTOCOL_ICMP)
342 if (get_icmp_o2i_ed_key (ip, &key))
345 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
347 udp = ip4_next_header (ip);
348 key.r_addr = ip->src_address;
349 key.l_addr = ip->dst_address;
350 key.proto = ip->protocol;
351 key.l_port = udp->dst_port;
352 key.r_port = udp->src_port;
356 key.r_addr = ip->src_address;
357 key.l_addr = ip->dst_address;
358 key.proto = ip->protocol;
359 key.l_port = key.r_port = 0;
362 kv.key[0] = key.as_u64[0];
363 kv.key[1] = key.as_u64[1];
365 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
367 s = pool_elt_at_index (tsm->sessions, value.value);
373 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
376 u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
380 nat_elog_warn ("create NAT user failed");
384 s = nat_ed_session_alloc (sm, u, thread_index, now);
387 nat44_delete_user_with_no_session (sm, u, thread_index);
388 nat_elog_warn ("create NAT session failed");
392 proto = ip_proto_to_snat_proto (key.proto);
394 s->ext_host_addr = key.r_addr;
395 s->ext_host_port = key.r_port;
396 s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
397 s->out2in.addr = key.l_addr;
398 s->out2in.port = key.l_port;
399 s->out2in.protocol = proto;
402 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
403 s->out2in.port = ip->protocol;
405 s->out2in.fib_index = 0;
406 s->in2out = s->out2in;
407 user_session_increment (sm, u, 0);
409 kv.value = s - tsm->sessions;
410 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
411 nat_elog_notice ("in2out_ed key add failed");
414 if (ip->protocol == IP_PROTOCOL_TCP)
416 tcp_header_t *tcp = ip4_next_header (ip);
417 if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
422 nat44_session_update_counters (s, now, 0, thread_index);
423 /* Per-user LRU list maintenance */
424 nat44_session_update_lru (sm, s, thread_index);
428 create_bypass_for_fwd_worker (snat_main_t * sm, ip4_header_t * ip,
431 ip4_header_t ip_wkr = {
432 .src_address = ip->dst_address,
434 u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
436 create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
439 #ifndef CLIB_MARCH_VARIANT
441 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
442 u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
443 u8 * p_proto, snat_session_key_t * p_value,
444 u8 * p_dont_translate, void *d, void *e)
446 u32 next = ~0, sw_if_index, rx_fib_index;
447 icmp46_header_t *icmp;
448 nat_ed_ses_key_t key;
449 clib_bihash_kv_16_8_t kv, value;
450 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
451 snat_session_t *s = 0;
452 u8 dont_translate = 0, is_addr_only, identity_nat;
453 snat_session_key_t e_key, l_key;
455 icmp = (icmp46_header_t *) ip4_next_header (ip);
456 sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
457 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
459 if (get_icmp_o2i_ed_key (ip, &key))
461 b->error = node->errors[NAT_OUT2IN_ED_ERROR_UNSUPPORTED_PROTOCOL];
462 next = NAT_NEXT_DROP;
465 key.fib_index = rx_fib_index;
466 kv.key[0] = key.as_u64[0];
467 kv.key[1] = key.as_u64[1];
469 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
471 /* Try to match static mapping */
472 e_key.addr = ip->dst_address;
473 e_key.port = key.l_port;
474 e_key.protocol = ip_proto_to_snat_proto (key.proto);
475 e_key.fib_index = rx_fib_index;
476 if (snat_static_mapping_match
477 (sm, e_key, &l_key, 1, &is_addr_only, 0, 0, 0, &identity_nat))
479 if (!sm->forwarding_enabled)
481 /* Don't NAT packet aimed at the intfc address */
482 if (PREDICT_FALSE (is_interface_addr (sm, node, sw_if_index,
483 ip->dst_address.as_u32)))
488 b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
489 next = NAT_NEXT_DROP;
495 if (next_src_nat (sm, ip, key.proto, key.l_port, key.r_port,
496 thread_index, rx_fib_index))
498 next = NAT_NEXT_IN2OUT_ED_FAST_PATH;
501 if (sm->num_workers > 1)
502 create_bypass_for_fwd_worker (sm, ip, rx_fib_index);
504 create_bypass_for_fwd (sm, ip, rx_fib_index, thread_index);
509 if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
510 (icmp->type != ICMP4_echo_request || !is_addr_only)))
512 b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
513 next = NAT_NEXT_DROP;
517 if (PREDICT_FALSE (identity_nat))
523 /* Create session initiated by host from external network */
524 s = create_session_for_static_mapping_ed (sm, b, l_key, e_key, node,
531 next = NAT_NEXT_DROP;
537 if (PREDICT_FALSE (icmp->type != ICMP4_echo_reply &&
538 icmp->type != ICMP4_echo_request &&
539 !icmp_is_error_message (icmp)))
541 b->error = node->errors[NAT_OUT2IN_ED_ERROR_BAD_ICMP_TYPE];
542 next = NAT_NEXT_DROP;
546 s = pool_elt_at_index (tsm->sessions, value.value);
549 *p_proto = ip_proto_to_snat_proto (key.proto);
552 *p_value = s->in2out;
553 *p_dont_translate = dont_translate;
555 *(snat_session_t **) d = s;
560 static snat_session_t *
561 nat44_ed_out2in_unknown_proto (snat_main_t * sm,
567 vlib_main_t * vm, vlib_node_runtime_t * node)
569 clib_bihash_kv_8_8_t kv, value;
570 clib_bihash_kv_16_8_t s_kv, s_value;
571 snat_static_mapping_t *m;
572 u32 old_addr, new_addr;
575 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
578 old_addr = ip->dst_address.as_u32;
580 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
583 if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
585 s = pool_elt_at_index (tsm->sessions, s_value.value);
586 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
590 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
592 b->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_SESSIONS_EXCEEDED];
593 nat_elog_notice ("maximum sessions exceeded");
597 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
598 if (clib_bihash_search_8_8
599 (&sm->static_mapping_by_external, &kv, &value))
601 b->error = node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
605 m = pool_elt_at_index (sm->static_mappings, value.value);
607 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
609 u = nat_user_get_or_create (sm, &m->local_addr, m->fib_index,
613 nat_elog_warn ("create NAT user failed");
617 /* Create a new session */
618 s = nat_ed_session_alloc (sm, u, thread_index, now);
621 nat44_delete_user_with_no_session (sm, u, thread_index);
622 nat_elog_warn ("create NAT session failed");
626 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
627 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
628 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
629 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
630 s->out2in.addr.as_u32 = old_addr;
631 s->out2in.fib_index = rx_fib_index;
632 s->in2out.addr.as_u32 = new_addr;
633 s->in2out.fib_index = m->fib_index;
634 s->in2out.port = s->out2in.port = ip->protocol;
635 user_session_increment (sm, u, 1);
637 /* Add to lookup tables */
638 s_kv.value = s - tsm->sessions;
639 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
640 nat_elog_notice ("out2in key add failed");
642 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
644 s_kv.value = s - tsm->sessions;
645 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
646 nat_elog_notice ("in2out key add failed");
649 /* Update IP checksum */
651 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
652 ip->checksum = ip_csum_fold (sum);
654 vnet_buffer (b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
657 nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
659 /* Per-user LRU list maintenance */
660 nat44_session_update_lru (sm, s, thread_index);
666 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
667 vlib_node_runtime_t * node,
668 vlib_frame_t * frame, int is_slow_path)
670 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
671 nat_next_t next_index;
672 snat_main_t *sm = &snat_main;
673 f64 now = vlib_time_now (vm);
674 u32 thread_index = vm->thread_index;
675 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
676 u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
679 stats_node_index = is_slow_path ? sm->ed_out2in_slowpath_node_index :
680 sm->ed_out2in_node_index;
682 from = vlib_frame_vector_args (frame);
683 n_left_from = frame->n_vectors;
684 next_index = node->cached_next_index;
686 while (n_left_from > 0)
690 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
692 while (n_left_from >= 4 && n_left_to_next >= 2)
695 vlib_buffer_t *b0, *b1;
696 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
698 u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1,
700 u16 old_port0, new_port0, old_port1, new_port1;
701 ip4_header_t *ip0, *ip1;
702 udp_header_t *udp0, *udp1;
703 tcp_header_t *tcp0, *tcp1;
704 icmp46_header_t *icmp0, *icmp1;
705 snat_session_t *s0 = 0, *s1 = 0;
706 clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
707 ip_csum_t sum0, sum1;
708 snat_session_key_t e_key0, l_key0, e_key1, l_key1;
709 lb_nat_type_t lb_nat0, lb_nat1;
710 twice_nat_type_t twice_nat0, twice_nat1;
711 u8 identity_nat0, identity_nat1;
713 /* Prefetch next iteration. */
715 vlib_buffer_t *p2, *p3;
717 p2 = vlib_get_buffer (vm, from[2]);
718 p3 = vlib_get_buffer (vm, from[3]);
720 vlib_prefetch_buffer_header (p2, LOAD);
721 vlib_prefetch_buffer_header (p3, LOAD);
723 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
724 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
727 /* speculatively enqueue b0 and b1 to the current next frame */
728 to_next[0] = bi0 = from[0];
729 to_next[1] = bi1 = from[1];
735 b0 = vlib_get_buffer (vm, bi0);
736 b1 = vlib_get_buffer (vm, bi1);
738 next0 = nat_buffer_opaque (b0)->arc_next;
739 next1 = nat_buffer_opaque (b1)->arc_next;
741 vnet_buffer (b0)->snat.flags = 0;
742 ip0 = vlib_buffer_get_current (b0);
744 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
746 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
749 if (PREDICT_FALSE (ip0->ttl == 1))
751 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
752 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
753 ICMP4_time_exceeded_ttl_exceeded_in_transit,
755 next0 = NAT_NEXT_ICMP_ERROR;
759 udp0 = ip4_next_header (ip0);
760 tcp0 = (tcp_header_t *) udp0;
761 icmp0 = (icmp46_header_t *) udp0;
762 proto0 = ip_proto_to_snat_proto (ip0->protocol);
766 if (PREDICT_FALSE (proto0 == ~0))
769 nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
770 thread_index, now, vm,
773 if (!sm->forwarding_enabled)
776 next0 = NAT_NEXT_DROP;
781 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
783 next0 = icmp_out2in_ed_slow_path
784 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
785 next0, now, thread_index, &s0);
792 if (PREDICT_FALSE (proto0 == ~0))
794 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
798 if (ip4_is_fragment (ip0))
800 next0 = NAT_NEXT_OUT2IN_ED_REASS;
805 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
807 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
812 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
813 ip0->protocol, rx_fib_index0, udp0->dst_port,
816 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
820 /* Try to match static mapping by external address and port,
821 destination address and port in packet */
822 e_key0.addr = ip0->dst_address;
823 e_key0.port = udp0->dst_port;
824 e_key0.protocol = proto0;
825 e_key0.fib_index = rx_fib_index0;
826 if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
827 &twice_nat0, &lb_nat0,
832 * Send DHCP packets to the ipv4 stack, or we won't
833 * be able to use dhcp client on the outside interface
835 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
836 && (udp0->dst_port ==
838 (UDP_DST_PORT_dhcp_to_client))))
843 if (!sm->forwarding_enabled)
846 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
847 next0 = NAT_NEXT_DROP;
851 if (next_src_nat (sm, ip0, ip0->protocol,
852 udp0->src_port, udp0->dst_port,
853 thread_index, rx_fib_index0))
855 next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
858 if (sm->num_workers > 1)
859 create_bypass_for_fwd_worker (sm, ip0,
862 create_bypass_for_fwd (sm, ip0, rx_fib_index0,
868 if (PREDICT_FALSE (identity_nat0))
871 if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
873 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
874 next0 = NAT_NEXT_DROP;
878 /* Create session initiated by host from external network */
879 s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
887 next0 = NAT_NEXT_DROP;
893 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
899 s0 = pool_elt_at_index (tsm->sessions, value0.value);
902 old_addr0 = ip0->dst_address.as_u32;
903 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
904 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
906 sum0 = ip0->checksum;
907 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
909 if (PREDICT_FALSE (is_twice_nat_session (s0)))
910 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
911 s0->ext_host_nat_addr.as_u32, ip4_header_t,
913 ip0->checksum = ip_csum_fold (sum0);
915 old_port0 = udp0->dst_port;
916 new_port0 = udp0->dst_port = s0->in2out.port;
918 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
920 sum0 = tcp0->checksum;
921 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
923 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
925 if (is_twice_nat_session (s0))
927 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
928 s0->ext_host_nat_addr.as_u32,
929 ip4_header_t, dst_address);
930 sum0 = ip_csum_update (sum0, tcp0->src_port,
931 s0->ext_host_nat_port, ip4_header_t,
933 tcp0->src_port = s0->ext_host_nat_port;
934 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
936 tcp0->checksum = ip_csum_fold (sum0);
938 if (nat44_set_tcp_session_state_o2i
939 (sm, s0, tcp0, thread_index))
942 else if (udp0->checksum)
944 sum0 = udp0->checksum;
945 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
947 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
949 if (PREDICT_FALSE (is_twice_nat_session (s0)))
951 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
952 s0->ext_host_nat_addr.as_u32,
953 ip4_header_t, dst_address);
954 sum0 = ip_csum_update (sum0, udp0->src_port,
955 s0->ext_host_nat_port, ip4_header_t,
957 udp0->src_port = s0->ext_host_nat_port;
958 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
960 udp0->checksum = ip_csum_fold (sum0);
965 if (PREDICT_FALSE (is_twice_nat_session (s0)))
967 udp0->src_port = s0->ext_host_nat_port;
968 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
974 nat44_session_update_counters (s0, now,
975 vlib_buffer_length_in_chain (vm, b0),
977 /* Per-user LRU list maintenance */
978 nat44_session_update_lru (sm, s0, thread_index);
981 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
982 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
984 nat44_ed_out2in_trace_t *t =
985 vlib_add_trace (vm, node, b0, sizeof (*t));
986 t->is_slow_path = is_slow_path;
987 t->sw_if_index = sw_if_index0;
988 t->next_index = next0;
989 t->session_index = ~0;
991 t->session_index = s0 - tsm->sessions;
994 pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
996 vnet_buffer (b1)->snat.flags = 0;
997 ip1 = vlib_buffer_get_current (b1);
999 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1001 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1004 if (PREDICT_FALSE (ip1->ttl == 1))
1006 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1007 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1008 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1010 next1 = NAT_NEXT_ICMP_ERROR;
1014 udp1 = ip4_next_header (ip1);
1015 tcp1 = (tcp_header_t *) udp1;
1016 icmp1 = (icmp46_header_t *) udp1;
1017 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1021 if (PREDICT_FALSE (proto1 == ~0))
1024 nat44_ed_out2in_unknown_proto (sm, b1, ip1, rx_fib_index1,
1025 thread_index, now, vm,
1028 if (!sm->forwarding_enabled)
1031 next1 = NAT_NEXT_DROP;
1036 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1038 next1 = icmp_out2in_ed_slow_path
1039 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1040 next1, now, thread_index, &s1);
1047 if (PREDICT_FALSE (proto1 == ~0))
1049 next1 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1053 if (ip4_is_fragment (ip1))
1055 next1 = NAT_NEXT_OUT2IN_ED_REASS;
1060 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1062 next1 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1067 make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address,
1068 ip1->protocol, rx_fib_index1, udp1->dst_port,
1071 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
1075 /* Try to match static mapping by external address and port,
1076 destination address and port in packet */
1077 e_key1.addr = ip1->dst_address;
1078 e_key1.port = udp1->dst_port;
1079 e_key1.protocol = proto1;
1080 e_key1.fib_index = rx_fib_index1;
1081 if (snat_static_mapping_match (sm, e_key1, &l_key1, 1, 0,
1082 &twice_nat1, &lb_nat1,
1087 * Send DHCP packets to the ipv4 stack, or we won't
1088 * be able to use dhcp client on the outside interface
1090 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1091 && (udp1->dst_port ==
1092 clib_host_to_net_u16
1093 (UDP_DST_PORT_dhcp_to_client))))
1098 if (!sm->forwarding_enabled)
1101 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1102 next1 = NAT_NEXT_DROP;
1106 if (next_src_nat (sm, ip1, ip1->protocol,
1107 udp1->src_port, udp1->dst_port,
1108 thread_index, rx_fib_index1))
1110 next1 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1113 if (sm->num_workers > 1)
1114 create_bypass_for_fwd_worker (sm, ip1,
1117 create_bypass_for_fwd (sm, ip1, rx_fib_index1,
1123 if (PREDICT_FALSE (identity_nat1))
1126 if ((proto1 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp1))
1128 b1->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1129 next1 = NAT_NEXT_DROP;
1133 /* Create session initiated by host from external network */
1134 s1 = create_session_for_static_mapping_ed (sm, b1, l_key1,
1142 next1 = NAT_NEXT_DROP;
1148 next1 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1154 s1 = pool_elt_at_index (tsm->sessions, value1.value);
1157 old_addr1 = ip1->dst_address.as_u32;
1158 new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
1159 vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1161 sum1 = ip1->checksum;
1162 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1164 if (PREDICT_FALSE (is_twice_nat_session (s1)))
1165 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1166 s1->ext_host_nat_addr.as_u32, ip4_header_t,
1168 ip1->checksum = ip_csum_fold (sum1);
1170 old_port1 = udp1->dst_port;
1171 new_port1 = udp1->dst_port = s1->in2out.port;
1173 if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1175 sum1 = tcp1->checksum;
1176 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1178 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1180 if (is_twice_nat_session (s1))
1182 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1183 s1->ext_host_nat_addr.as_u32,
1184 ip4_header_t, dst_address);
1185 sum1 = ip_csum_update (sum1, tcp1->src_port,
1186 s1->ext_host_nat_port, ip4_header_t,
1188 tcp1->src_port = s1->ext_host_nat_port;
1189 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1191 tcp1->checksum = ip_csum_fold (sum1);
1193 if (nat44_set_tcp_session_state_o2i
1194 (sm, s1, tcp1, thread_index))
1197 else if (udp1->checksum)
1199 sum1 = udp1->checksum;
1200 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1202 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1204 if (PREDICT_FALSE (is_twice_nat_session (s1)))
1206 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
1207 s1->ext_host_nat_addr.as_u32,
1208 ip4_header_t, dst_address);
1209 sum1 = ip_csum_update (sum1, udp1->src_port,
1210 s1->ext_host_nat_port, ip4_header_t,
1212 udp1->src_port = s1->ext_host_nat_port;
1213 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1215 udp1->checksum = ip_csum_fold (sum1);
1220 if (PREDICT_FALSE (is_twice_nat_session (s1)))
1222 udp1->src_port = s1->ext_host_nat_port;
1223 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
1229 nat44_session_update_counters (s1, now,
1230 vlib_buffer_length_in_chain (vm, b1),
1232 /* Per-user LRU list maintenance */
1233 nat44_session_update_lru (sm, s1, thread_index);
1236 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1237 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1239 nat44_ed_out2in_trace_t *t =
1240 vlib_add_trace (vm, node, b1, sizeof (*t));
1241 t->is_slow_path = is_slow_path;
1242 t->sw_if_index = sw_if_index1;
1243 t->next_index = next1;
1244 t->session_index = ~0;
1246 t->session_index = s1 - tsm->sessions;
1249 pkts_processed += next1 == nat_buffer_opaque (b1)->arc_next;
1251 /* verify speculative enqueues, maybe switch current next frame */
1252 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1253 to_next, n_left_to_next,
1254 bi0, bi1, next0, next1);
1257 while (n_left_from > 0 && n_left_to_next > 0)
1261 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0,
1263 u16 old_port0, new_port0;
1267 icmp46_header_t *icmp0;
1268 snat_session_t *s0 = 0;
1269 clib_bihash_kv_16_8_t kv0, value0;
1271 snat_session_key_t e_key0, l_key0;
1272 lb_nat_type_t lb_nat0;
1273 twice_nat_type_t twice_nat0;
1276 /* speculatively enqueue b0 to the current next frame */
1282 n_left_to_next -= 1;
1284 b0 = vlib_get_buffer (vm, bi0);
1285 next0 = nat_buffer_opaque (b0)->arc_next;
1287 vnet_buffer (b0)->snat.flags = 0;
1288 ip0 = vlib_buffer_get_current (b0);
1290 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1292 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1295 if (PREDICT_FALSE (ip0->ttl == 1))
1297 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1298 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1299 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1301 next0 = NAT_NEXT_ICMP_ERROR;
1305 udp0 = ip4_next_header (ip0);
1306 tcp0 = (tcp_header_t *) udp0;
1307 icmp0 = (icmp46_header_t *) udp0;
1308 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1312 if (PREDICT_FALSE (proto0 == ~0))
1315 nat44_ed_out2in_unknown_proto (sm, b0, ip0, rx_fib_index0,
1316 thread_index, now, vm,
1319 if (!sm->forwarding_enabled)
1322 next0 = NAT_NEXT_DROP;
1327 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1329 next0 = icmp_out2in_ed_slow_path
1330 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1331 next0, now, thread_index, &s0);
1338 if (PREDICT_FALSE (proto0 == ~0))
1340 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1344 if (ip4_is_fragment (ip0))
1346 next0 = NAT_NEXT_OUT2IN_ED_REASS;
1351 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1353 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1358 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1359 ip0->protocol, rx_fib_index0, udp0->dst_port,
1362 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
1366 /* Try to match static mapping by external address and port,
1367 destination address and port in packet */
1368 e_key0.addr = ip0->dst_address;
1369 e_key0.port = udp0->dst_port;
1370 e_key0.protocol = proto0;
1371 e_key0.fib_index = rx_fib_index0;
1372 if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1373 &twice_nat0, &lb_nat0,
1378 * Send DHCP packets to the ipv4 stack, or we won't
1379 * be able to use dhcp client on the outside interface
1381 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1382 && (udp0->dst_port ==
1383 clib_host_to_net_u16
1384 (UDP_DST_PORT_dhcp_to_client))))
1389 if (!sm->forwarding_enabled)
1392 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1393 next0 = NAT_NEXT_DROP;
1397 if (next_src_nat (sm, ip0, ip0->protocol,
1398 udp0->src_port, udp0->dst_port,
1399 thread_index, rx_fib_index0))
1401 next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1404 if (sm->num_workers > 1)
1405 create_bypass_for_fwd_worker (sm, ip0,
1408 create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1414 if (PREDICT_FALSE (identity_nat0))
1417 if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1419 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1420 next0 = NAT_NEXT_DROP;
1424 /* Create session initiated by host from external network */
1425 s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1433 next0 = NAT_NEXT_DROP;
1439 next0 = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
1445 s0 = pool_elt_at_index (tsm->sessions, value0.value);
1448 old_addr0 = ip0->dst_address.as_u32;
1449 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
1450 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1452 sum0 = ip0->checksum;
1453 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1455 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1456 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1457 s0->ext_host_nat_addr.as_u32, ip4_header_t,
1459 ip0->checksum = ip_csum_fold (sum0);
1461 old_port0 = udp0->dst_port;
1462 new_port0 = udp0->dst_port = s0->in2out.port;
1464 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1466 sum0 = tcp0->checksum;
1467 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1469 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1471 if (is_twice_nat_session (s0))
1473 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1474 s0->ext_host_nat_addr.as_u32,
1475 ip4_header_t, dst_address);
1476 sum0 = ip_csum_update (sum0, tcp0->src_port,
1477 s0->ext_host_nat_port, ip4_header_t,
1479 tcp0->src_port = s0->ext_host_nat_port;
1480 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1482 tcp0->checksum = ip_csum_fold (sum0);
1484 if (nat44_set_tcp_session_state_o2i
1485 (sm, s0, tcp0, thread_index))
1488 else if (udp0->checksum)
1490 sum0 = udp0->checksum;
1491 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1493 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1495 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1497 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1498 s0->ext_host_nat_addr.as_u32,
1499 ip4_header_t, dst_address);
1500 sum0 = ip_csum_update (sum0, udp0->src_port,
1501 s0->ext_host_nat_port, ip4_header_t,
1503 udp0->src_port = s0->ext_host_nat_port;
1504 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1506 udp0->checksum = ip_csum_fold (sum0);
1511 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1513 udp0->src_port = s0->ext_host_nat_port;
1514 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1520 nat44_session_update_counters (s0, now,
1521 vlib_buffer_length_in_chain (vm, b0),
1523 /* Per-user LRU list maintenance */
1524 nat44_session_update_lru (sm, s0, thread_index);
1527 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1528 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1530 nat44_ed_out2in_trace_t *t =
1531 vlib_add_trace (vm, node, b0, sizeof (*t));
1532 t->is_slow_path = is_slow_path;
1533 t->sw_if_index = sw_if_index0;
1534 t->next_index = next0;
1535 t->session_index = ~0;
1537 t->session_index = s0 - tsm->sessions;
1540 pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
1541 /* verify speculative enqueue, maybe switch current next frame */
1542 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1543 to_next, n_left_to_next,
1547 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1550 vlib_node_increment_counter (vm, stats_node_index,
1551 NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1553 vlib_node_increment_counter (vm, stats_node_index,
1554 NAT_OUT2IN_ED_ERROR_TCP_PACKETS, tcp_packets);
1555 vlib_node_increment_counter (vm, stats_node_index,
1556 NAT_OUT2IN_ED_ERROR_UDP_PACKETS, udp_packets);
1557 vlib_node_increment_counter (vm, stats_node_index,
1558 NAT_OUT2IN_ED_ERROR_ICMP_PACKETS,
1560 vlib_node_increment_counter (vm, stats_node_index,
1561 NAT_OUT2IN_ED_ERROR_OTHER_PACKETS,
1563 vlib_node_increment_counter (vm, stats_node_index,
1564 NAT_OUT2IN_ED_ERROR_FRAGMENTS, fragments);
1565 return frame->n_vectors;
1569 nat44_ed_out2in_reass_node_fn_inline (vlib_main_t * vm,
1570 vlib_node_runtime_t * node,
1571 vlib_frame_t * frame)
1573 u32 n_left_from, *from, *to_next;
1574 nat_next_t next_index;
1575 u32 pkts_processed = 0;
1576 snat_main_t *sm = &snat_main;
1577 f64 now = vlib_time_now (vm);
1578 u32 thread_index = vm->thread_index;
1579 snat_main_per_thread_data_t *per_thread_data =
1580 &sm->per_thread_data[thread_index];
1581 u32 *fragments_to_drop = 0;
1582 u32 *fragments_to_loopback = 0;
1584 from = vlib_frame_vector_args (frame);
1585 n_left_from = frame->n_vectors;
1586 next_index = node->cached_next_index;
1588 while (n_left_from > 0)
1592 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1594 while (n_left_from > 0 && n_left_to_next > 0)
1596 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1601 nat_reass_ip4_t *reass0;
1604 icmp46_header_t *icmp0;
1605 clib_bihash_kv_16_8_t kv0, value0;
1606 snat_session_t *s0 = 0;
1607 u16 old_port0, new_port0;
1609 snat_session_key_t e_key0, l_key0;
1611 twice_nat_type_t twice_nat0;
1614 /* speculatively enqueue b0 to the current next frame */
1620 n_left_to_next -= 1;
1622 b0 = vlib_get_buffer (vm, bi0);
1623 next0 = nat_buffer_opaque (b0)->arc_next;
1625 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1627 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1630 if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1632 next0 = NAT_NEXT_DROP;
1633 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT];
1637 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1638 udp0 = ip4_next_header (ip0);
1639 tcp0 = (tcp_header_t *) udp0;
1640 icmp0 = (icmp46_header_t *) udp0;
1641 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1643 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1647 1, &fragments_to_drop);
1649 if (PREDICT_FALSE (!reass0))
1651 next0 = NAT_NEXT_DROP;
1652 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_REASS];
1653 nat_elog_notice ("maximum reassemblies exceeded");
1657 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1659 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1661 next0 = icmp_out2in_ed_slow_path
1662 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1663 next0, now, thread_index, &s0);
1665 if (PREDICT_TRUE (next0 != NAT_NEXT_DROP))
1668 reass0->sess_index = s0 - per_thread_data->sessions;
1670 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1671 reass0->thread_index = thread_index;
1672 nat_ip4_reass_get_frags (reass0,
1673 &fragments_to_loopback);
1679 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address,
1680 ip0->protocol, rx_fib_index0, udp0->dst_port,
1683 if (clib_bihash_search_16_8
1684 (&per_thread_data->out2in_ed, &kv0, &value0))
1686 /* Try to match static mapping by external address and port,
1687 destination address and port in packet */
1688 e_key0.addr = ip0->dst_address;
1689 e_key0.port = udp0->dst_port;
1690 e_key0.protocol = proto0;
1691 e_key0.fib_index = rx_fib_index0;
1692 if (snat_static_mapping_match (sm, e_key0, &l_key0, 1, 0,
1693 &twice_nat0, &lb0, 0,
1697 * Send DHCP packets to the ipv4 stack, or we won't
1698 * be able to use dhcp client on the outside interface
1700 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1703 clib_host_to_net_u16
1704 (UDP_DST_PORT_dhcp_to_client))))
1709 if (!sm->forwarding_enabled)
1712 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1713 next0 = NAT_NEXT_DROP;
1717 if (next_src_nat (sm, ip0, ip0->protocol,
1718 udp0->src_port, udp0->dst_port,
1719 thread_index, rx_fib_index0))
1721 next0 = NAT_NEXT_IN2OUT_ED_FAST_PATH;
1724 if (sm->num_workers > 1)
1725 create_bypass_for_fwd_worker (sm, ip0,
1728 create_bypass_for_fwd (sm, ip0, rx_fib_index0,
1730 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1731 nat_ip4_reass_get_frags (reass0,
1732 &fragments_to_loopback);
1737 if (PREDICT_FALSE (identity_nat0))
1739 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1743 if ((proto0 == SNAT_PROTOCOL_TCP) && !tcp_is_init (tcp0))
1745 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_NON_SYN];
1746 next0 = NAT_NEXT_DROP;
1750 /* Create session initiated by host from external network */
1751 s0 = create_session_for_static_mapping_ed (sm, b0, l_key0,
1759 node->errors[NAT_OUT2IN_ED_ERROR_NO_TRANSLATION];
1760 next0 = NAT_NEXT_DROP;
1763 reass0->sess_index = s0 - per_thread_data->sessions;
1764 reass0->thread_index = thread_index;
1768 s0 = pool_elt_at_index (per_thread_data->sessions,
1770 reass0->sess_index = value0.value;
1772 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1776 if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1778 if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1780 if (nat_ip4_reass_add_fragment
1781 (thread_index, reass0, bi0, &fragments_to_drop))
1783 b0->error = node->errors[NAT_OUT2IN_ED_ERROR_MAX_FRAG];
1785 ("maximum fragments per reassembly exceeded");
1786 next0 = NAT_NEXT_DROP;
1792 s0 = pool_elt_at_index (per_thread_data->sessions,
1793 reass0->sess_index);
1796 old_addr0 = ip0->dst_address.as_u32;
1797 ip0->dst_address = s0->in2out.addr;
1798 new_addr0 = ip0->dst_address.as_u32;
1799 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1801 sum0 = ip0->checksum;
1802 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1804 dst_address /* changed member */ );
1805 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1806 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1807 s0->ext_host_nat_addr.as_u32, ip4_header_t,
1809 ip0->checksum = ip_csum_fold (sum0);
1811 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1813 old_port0 = udp0->dst_port;
1814 new_port0 = udp0->dst_port = s0->in2out.port;
1816 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1818 sum0 = tcp0->checksum;
1819 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1821 dst_address /* changed member */ );
1823 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1824 ip4_header_t /* cheat */ ,
1825 length /* changed member */ );
1826 if (is_twice_nat_session (s0))
1828 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1829 s0->ext_host_nat_addr.as_u32,
1830 ip4_header_t, dst_address);
1831 sum0 = ip_csum_update (sum0, tcp0->src_port,
1832 s0->ext_host_nat_port,
1833 ip4_header_t, length);
1834 tcp0->src_port = s0->ext_host_nat_port;
1835 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1837 tcp0->checksum = ip_csum_fold (sum0);
1839 else if (udp0->checksum)
1841 sum0 = udp0->checksum;
1843 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1846 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1848 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1850 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
1851 s0->ext_host_nat_addr.as_u32,
1852 ip4_header_t, dst_address);
1853 sum0 = ip_csum_update (sum0, udp0->src_port,
1854 s0->ext_host_nat_port,
1855 ip4_header_t, length);
1856 udp0->src_port = s0->ext_host_nat_port;
1857 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1859 udp0->checksum = ip_csum_fold (sum0);
1863 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1865 udp0->src_port = s0->ext_host_nat_port;
1866 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
1872 nat44_session_update_counters (s0, now,
1873 vlib_buffer_length_in_chain (vm, b0),
1875 /* Per-user LRU list maintenance */
1876 nat44_session_update_lru (sm, s0, thread_index);
1879 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1880 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1882 nat44_reass_trace_t *t =
1883 vlib_add_trace (vm, node, b0, sizeof (*t));
1884 t->cached = cached0;
1885 t->sw_if_index = sw_if_index0;
1886 t->next_index = next0;
1896 pkts_processed += next0 != NAT_NEXT_DROP;
1898 /* verify speculative enqueue, maybe switch current next frame */
1899 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1900 to_next, n_left_to_next,
1904 if (n_left_from == 0 && vec_len (fragments_to_loopback))
1906 from = vlib_frame_vector_args (frame);
1907 u32 len = vec_len (fragments_to_loopback);
1908 if (len <= VLIB_FRAME_SIZE)
1910 clib_memcpy_fast (from, fragments_to_loopback,
1911 sizeof (u32) * len);
1913 vec_reset_length (fragments_to_loopback);
1917 clib_memcpy_fast (from, fragments_to_loopback +
1918 (len - VLIB_FRAME_SIZE),
1919 sizeof (u32) * VLIB_FRAME_SIZE);
1920 n_left_from = VLIB_FRAME_SIZE;
1921 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1926 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1929 vlib_node_increment_counter (vm, sm->ed_out2in_reass_node_index,
1930 NAT_OUT2IN_ED_ERROR_OUT2IN_PACKETS,
1933 nat_send_all_to_node (vm, fragments_to_drop, node,
1934 &node->errors[NAT_OUT2IN_ED_ERROR_DROP_FRAGMENT],
1937 vec_free (fragments_to_drop);
1938 vec_free (fragments_to_loopback);
1939 return frame->n_vectors;
1942 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
1943 vlib_node_runtime_t * node,
1944 vlib_frame_t * frame)
1946 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
1950 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
1951 .name = "nat44-ed-out2in",
1952 .vector_size = sizeof (u32),
1953 .sibling_of = "nat-default",
1954 .format_trace = format_nat44_ed_out2in_trace,
1955 .type = VLIB_NODE_TYPE_INTERNAL,
1956 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1957 .error_strings = nat_out2in_ed_error_strings,
1958 .runtime_data_bytes = sizeof (snat_runtime_t),
1962 VLIB_NODE_FN (nat44_ed_out2in_slowpath_node) (vlib_main_t * vm,
1963 vlib_node_runtime_t * node,
1964 vlib_frame_t * frame)
1966 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
1970 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
1971 .name = "nat44-ed-out2in-slowpath",
1972 .vector_size = sizeof (u32),
1973 .sibling_of = "nat-default",
1974 .format_trace = format_nat44_ed_out2in_trace,
1975 .type = VLIB_NODE_TYPE_INTERNAL,
1976 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1977 .error_strings = nat_out2in_ed_error_strings,
1978 .runtime_data_bytes = sizeof (snat_runtime_t),
1982 VLIB_NODE_FN (nat44_ed_out2in_reass_node) (vlib_main_t * vm,
1983 vlib_node_runtime_t * node,
1984 vlib_frame_t * frame)
1986 return nat44_ed_out2in_reass_node_fn_inline (vm, node, frame);
1990 VLIB_REGISTER_NODE (nat44_ed_out2in_reass_node) = {
1991 .name = "nat44-ed-out2in-reass",
1992 .vector_size = sizeof (u32),
1993 .sibling_of = "nat-default",
1994 .format_trace = format_nat44_reass_trace,
1995 .type = VLIB_NODE_TYPE_INTERNAL,
1996 .n_errors = ARRAY_LEN(nat_out2in_ed_error_strings),
1997 .error_strings = nat_out2in_ed_error_strings,
2002 format_nat_pre_trace (u8 * s, va_list * args)
2004 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2005 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2006 nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
2007 return format (s, "out2in next_index %d", t->next_index);
2010 VLIB_NODE_FN (nat_pre_out2in_node) (vlib_main_t * vm,
2011 vlib_node_runtime_t * node,
2012 vlib_frame_t * frame)
2014 return nat_pre_node_fn_inline (vm, node, frame,
2015 NAT_NEXT_OUT2IN_ED_FAST_PATH);
2019 VLIB_REGISTER_NODE (nat_pre_out2in_node) = {
2020 .name = "nat-pre-out2in",
2021 .vector_size = sizeof (u32),
2022 .sibling_of = "nat-default",
2023 .format_trace = format_nat_pre_trace,
2024 .type = VLIB_NODE_TYPE_INTERNAL,
2030 * fd.io coding-style-patch-verification: ON
2033 * eval: (c-set-style "gnu")