2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
17 * @brief NAT44 endpoint-dependent inside to outside network translation
20 #include <vlib/vlib.h>
21 #include <vnet/vnet.h>
22 #include <vnet/pg/pg.h>
23 #include <vnet/ip/ip.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/fib/ip4_fib.h>
26 #include <vnet/udp/udp.h>
27 #include <vppinfra/error.h>
29 #include <nat/nat_ipfix_logging.h>
30 #include <nat/nat_reass.h>
31 #include <nat/nat_inlines.h>
32 #include <nat/nat_syslog.h>
33 #include <nat/nat_ha.h>
35 static char *nat_in2out_ed_error_strings[] = {
36 #define _(sym,string) string,
37 foreach_nat_in2out_ed_error
47 } nat_in2out_ed_trace_t;
50 format_nat_in2out_ed_trace (u8 * s, va_list * args)
52 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
53 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
54 nat_in2out_ed_trace_t *t = va_arg (*args, nat_in2out_ed_trace_t *);
58 t->is_slow_path ? "NAT44_IN2OUT_ED_SLOW_PATH" :
59 "NAT44_IN2OUT_ED_FAST_PATH";
61 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
62 t->sw_if_index, t->next_index, t->session_index);
67 #ifndef CLIB_MARCH_VARIANT
69 nat44_i2o_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void *arg)
71 snat_main_t *sm = &snat_main;
72 nat44_is_idle_session_ctx_t *ctx = arg;
74 u64 sess_timeout_time;
75 nat_ed_ses_key_t ed_key;
76 clib_bihash_kv_16_8_t ed_kv;
79 snat_session_key_t key;
80 snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
83 s = pool_elt_at_index (tsm->sessions, kv->value);
84 sess_timeout_time = s->last_heard + (f64) nat44_session_get_timeout (sm, s);
85 if (ctx->now >= sess_timeout_time)
87 if (is_fwd_bypass_session (s))
90 ed_key.l_addr = s->out2in.addr;
91 ed_key.r_addr = s->ext_host_addr;
92 ed_key.fib_index = s->out2in.fib_index;
93 if (snat_is_unk_proto_session (s))
95 ed_key.proto = s->in2out.port;
101 ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
102 ed_key.l_port = s->out2in.port;
103 ed_key.r_port = s->ext_host_port;
105 ed_kv.key[0] = ed_key.as_u64[0];
106 ed_kv.key[1] = ed_key.as_u64[1];
107 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &ed_kv, 0))
108 nat_elog_warn ("out2in_ed key del failed");
110 if (snat_is_unk_proto_session (s))
113 snat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
114 s->in2out.addr.as_u32,
115 s->out2in.addr.as_u32,
119 s->in2out.fib_index);
121 nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
122 &s->in2out.addr, s->in2out.port,
123 &s->ext_host_nat_addr, s->ext_host_nat_port,
124 &s->out2in.addr, s->out2in.port,
125 &s->ext_host_addr, s->ext_host_port,
126 s->in2out.protocol, is_twice_nat_session (s));
128 nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
129 s->ext_host_port, s->out2in.protocol, s->out2in.fib_index,
132 if (is_twice_nat_session (s))
134 for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
136 key.protocol = s->in2out.protocol;
137 key.port = s->ext_host_nat_port;
138 a = sm->twice_nat_addresses + i;
139 if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
141 snat_free_outside_address_and_port (sm->twice_nat_addresses,
149 if (snat_is_session_static (s))
152 snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
155 nat44_delete_session (sm, s, ctx->thread_index);
164 icmp_in2out_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
165 ip4_header_t * ip0, icmp46_header_t * icmp0,
166 u32 sw_if_index0, u32 rx_fib_index0,
167 vlib_node_runtime_t * node, u32 next0, f64 now,
168 u32 thread_index, snat_session_t ** p_s0)
170 next0 = icmp_in2out (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
171 next0, thread_index, p_s0, 0);
172 snat_session_t *s0 = *p_s0;
173 if (PREDICT_TRUE (next0 != NAT_NEXT_DROP && s0))
176 nat44_session_update_counters (s0, now,
177 vlib_buffer_length_in_chain
178 (sm->vlib_main, b0), thread_index);
179 /* Per-user LRU list maintenance */
180 nat44_session_update_lru (sm, s0, thread_index);
186 slow_path_ed (snat_main_t * sm,
189 clib_bihash_kv_16_8_t * kv,
190 snat_session_t ** sessionp,
191 vlib_node_runtime_t * node, u32 next, u32 thread_index, f64 now,
194 snat_session_t *s = 0;
196 snat_session_key_t key0, key1;
197 lb_nat_type_t lb = 0, is_sm = 0;
198 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
199 nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
200 u32 proto = ip_proto_to_snat_proto (key->proto);
201 nat_outside_fib_t *outside_fib;
202 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
205 .fp_proto = FIB_PROTOCOL_IP4,
208 .ip4.as_u32 = key->r_addr.as_u32,
211 nat44_is_idle_session_ctx_t ctx;
213 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
215 b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
216 nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
217 nat_elog_notice ("maximum sessions exceeded");
218 return NAT_NEXT_DROP;
221 key0.addr = key->l_addr;
222 key0.port = key->l_port;
223 key1.protocol = key0.protocol = proto;
224 key0.fib_index = rx_fib_index;
225 key1.fib_index = sm->outside_fib_index;
226 /* First try to match static mapping by local address and port */
227 if (snat_static_mapping_match
228 (sm, key0, &key1, 0, 0, 0, &lb, 0, &identity_nat))
230 /* Try to create dynamic translation */
231 if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
234 tsm->snat_thread_index))
236 nat_elog_notice ("addresses exhausted");
237 b->error = node->errors[NAT_IN2OUT_ED_ERROR_OUT_OF_PORTS];
238 return NAT_NEXT_DROP;
243 if (PREDICT_FALSE (identity_nat))
252 if (proto == SNAT_PROTOCOL_TCP)
254 if (!tcp_is_init (tcp))
256 b->error = node->errors[NAT_IN2OUT_ED_ERROR_NON_SYN];
257 return NAT_NEXT_DROP;
261 u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
264 nat_elog_warn ("create NAT user failed");
266 snat_free_outside_address_and_port (sm->addresses,
267 thread_index, &key1);
268 return NAT_NEXT_DROP;
271 s = nat_ed_session_alloc (sm, u, thread_index, now);
274 nat44_delete_user_with_no_session (sm, u, thread_index);
275 nat_elog_warn ("create NAT session failed");
277 snat_free_outside_address_and_port (sm->addresses,
278 thread_index, &key1);
279 return NAT_NEXT_DROP;
282 user_session_increment (sm, u, is_sm);
284 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
286 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
287 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
288 s->ext_host_addr = key->r_addr;
289 s->ext_host_port = key->r_port;
292 s->out2in.protocol = key0.protocol;
294 switch (vec_len (sm->outside_fibs))
297 s->out2in.fib_index = sm->outside_fib_index;
300 s->out2in.fib_index = sm->outside_fibs[0].fib_index;
304 vec_foreach (outside_fib, sm->outside_fibs)
306 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
307 if (FIB_NODE_INDEX_INVALID != fei)
309 if (fib_entry_get_resolving_interface (fei) != ~0)
311 s->out2in.fib_index = outside_fib->fib_index;
320 /* Add to lookup tables */
321 kv->value = s - tsm->sessions;
323 ctx.thread_index = thread_index;
324 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, kv,
325 nat44_i2o_ed_is_idle_session_cb,
327 nat_elog_notice ("in2out-ed key add failed");
329 make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
330 key1.port, key->r_port);
331 kv->value = s - tsm->sessions;
332 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, kv,
333 nat44_o2i_ed_is_idle_session_cb,
335 nat_elog_notice ("out2in-ed key add failed");
340 snat_ipfix_logging_nat44_ses_create (thread_index,
341 s->in2out.addr.as_u32,
342 s->out2in.addr.as_u32,
345 s->out2in.port, s->in2out.fib_index);
347 nat_syslog_nat44_sadd (s->user_index, s->in2out.fib_index,
348 &s->in2out.addr, s->in2out.port,
349 &s->ext_host_nat_addr, s->ext_host_nat_port,
350 &s->out2in.addr, s->out2in.port,
351 &s->ext_host_addr, s->ext_host_port,
352 s->in2out.protocol, 0);
354 nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
355 s->out2in.port, &s->ext_host_addr, s->ext_host_port,
356 &s->ext_host_nat_addr, s->ext_host_nat_port,
357 s->in2out.protocol, s->in2out.fib_index, s->flags,
363 static_always_inline int
364 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t * node,
365 u32 sw_if_index, ip4_header_t * ip, u32 proto,
366 u32 rx_fib_index, u32 thread_index)
368 udp_header_t *udp = ip4_next_header (ip);
369 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
370 clib_bihash_kv_16_8_t kv, value;
371 snat_session_key_t key0, key1;
373 make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
374 sm->outside_fib_index, udp->dst_port, udp->src_port);
376 /* NAT packet aimed at external address if */
377 /* has active sessions */
378 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
380 key0.addr = ip->dst_address;
381 key0.port = udp->dst_port;
382 key0.protocol = proto;
383 key0.fib_index = sm->outside_fib_index;
384 /* or is static mappings */
385 if (!snat_static_mapping_match (sm, key0, &key1, 1, 0, 0, 0, 0, 0))
391 if (sm->forwarding_enabled)
394 return snat_not_translate_fast (sm, node, sw_if_index, ip, proto,
398 static_always_inline int
399 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
400 u32 thread_index, f64 now,
401 vlib_main_t * vm, vlib_buffer_t * b)
403 nat_ed_ses_key_t key;
404 clib_bihash_kv_16_8_t kv, value;
406 snat_session_t *s = 0;
407 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
409 if (!sm->forwarding_enabled)
412 if (ip->protocol == IP_PROTOCOL_ICMP)
414 key.as_u64[0] = key.as_u64[1] = 0;
415 if (get_icmp_i2o_ed_key (ip, &key))
418 kv.key[0] = key.as_u64[0];
419 kv.key[1] = key.as_u64[1];
421 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
423 udp = ip4_next_header (ip);
424 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
425 udp->src_port, udp->dst_port);
429 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
433 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
435 s = pool_elt_at_index (tsm->sessions, value.value);
436 if (is_fwd_bypass_session (s))
438 if (ip->protocol == IP_PROTOCOL_TCP)
440 tcp_header_t *tcp = ip4_next_header (ip);
441 if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
445 nat44_session_update_counters (s, now,
446 vlib_buffer_length_in_chain (vm, b),
448 /* Per-user LRU list maintenance */
449 nat44_session_update_lru (sm, s, thread_index);
459 static_always_inline int
460 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
461 u8 proto, u16 src_port, u16 dst_port,
462 u32 thread_index, u32 rx_sw_if_index,
465 clib_bihash_kv_16_8_t kv, value;
466 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
469 u32 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (rx_sw_if_index);
470 u32 tx_fib_index = ip4_fib_table_get_index_for_sw_if_index (tx_sw_if_index);
473 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, tx_fib_index,
475 if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
477 s = pool_elt_at_index (tsm->sessions, value.value);
478 if (nat44_is_ses_closed (s))
480 nat_free_session_data (sm, s, thread_index, 0);
481 nat44_delete_session (sm, s, thread_index);
484 s->flags |= SNAT_SESSION_FLAG_OUTPUT_FEATURE;
489 make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, rx_fib_index,
491 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
493 s = pool_elt_at_index (tsm->sessions, value.value);
494 if (is_fwd_bypass_session (s))
499 pool_foreach (i, sm->output_feature_interfaces,
501 if ((nat_interface_is_inside (i)) && (rx_sw_if_index == i->sw_if_index))
511 #ifndef CLIB_MARCH_VARIANT
513 icmp_match_in2out_ed (snat_main_t * sm, vlib_node_runtime_t * node,
514 u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
515 u8 * p_proto, snat_session_key_t * p_value,
516 u8 * p_dont_translate, void *d, void *e)
518 icmp46_header_t *icmp;
521 nat_ed_ses_key_t key;
522 snat_session_t *s = 0;
523 u8 dont_translate = 0;
524 clib_bihash_kv_16_8_t kv, value;
527 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
529 icmp = (icmp46_header_t *) ip4_next_header (ip);
530 sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
531 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
533 key.as_u64[0] = key.as_u64[1] = 0;
534 err = get_icmp_i2o_ed_key (ip, &key);
537 b->error = node->errors[err];
538 next = NAT_NEXT_DROP;
541 key.fib_index = rx_fib_index;
543 kv.key[0] = key.as_u64[0];
544 kv.key[1] = key.as_u64[1];
546 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
548 if (vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0)
550 if (PREDICT_FALSE (nat44_ed_not_translate_output_feature (sm, ip,
569 if (PREDICT_FALSE (nat44_ed_not_translate (sm, node, sw_if_index,
570 ip, SNAT_PROTOCOL_ICMP,
579 if (PREDICT_FALSE (icmp_is_error_message (icmp)))
581 b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
582 next = NAT_NEXT_DROP;
586 next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
587 thread_index, vlib_time_now (sm->vlib_main), 0);
589 if (PREDICT_FALSE (next == NAT_NEXT_DROP))
600 if (PREDICT_FALSE (icmp->type != ICMP4_echo_request &&
601 icmp->type != ICMP4_echo_reply &&
602 !icmp_is_error_message (icmp)))
604 b->error = node->errors[NAT_IN2OUT_ED_ERROR_BAD_ICMP_TYPE];
605 next = NAT_NEXT_DROP;
609 s = pool_elt_at_index (tsm->sessions, value.value);
612 *p_proto = ip_proto_to_snat_proto (key.proto);
615 *p_value = s->out2in;
616 *p_dont_translate = dont_translate;
618 *(snat_session_t **) d = s;
623 static snat_session_t *
624 nat44_ed_in2out_unknown_proto (snat_main_t * sm,
630 vlib_main_t * vm, vlib_node_runtime_t * node)
632 clib_bihash_kv_8_8_t kv, value;
633 clib_bihash_kv_16_8_t s_kv, s_value;
634 snat_static_mapping_t *m;
635 u32 old_addr, new_addr = 0;
638 dlist_elt_t *head, *elt;
639 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
640 u32 elt_index, head_index, ses_index;
642 u32 outside_fib_index = sm->outside_fib_index;
645 nat_outside_fib_t *outside_fib;
646 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
648 .fp_proto = FIB_PROTOCOL_IP4,
651 .ip4.as_u32 = ip->dst_address.as_u32,
655 switch (vec_len (sm->outside_fibs))
658 outside_fib_index = sm->outside_fib_index;
661 outside_fib_index = sm->outside_fibs[0].fib_index;
665 vec_foreach (outside_fib, sm->outside_fibs)
667 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
668 if (FIB_NODE_INDEX_INVALID != fei)
670 if (fib_entry_get_resolving_interface (fei) != ~0)
672 outside_fib_index = outside_fib->fib_index;
680 old_addr = ip->src_address.as_u32;
682 make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
685 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
687 s = pool_elt_at_index (tsm->sessions, s_value.value);
688 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
692 if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
694 b->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_SESSIONS_EXCEEDED];
695 nat_ipfix_logging_max_sessions (thread_index, sm->max_translations);
696 nat_elog_notice ("maximum sessions exceeded");
700 u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
704 nat_elog_warn ("create NAT user failed");
708 make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
710 /* Try to find static mapping first */
711 if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
713 m = pool_elt_at_index (sm->static_mappings, value.value);
714 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
718 /* Fallback to 3-tuple key */
721 /* Choose same out address as for TCP/UDP session to same destination */
722 head_index = u->sessions_per_user_list_head_index;
723 head = pool_elt_at_index (tsm->list_pool, head_index);
724 elt_index = head->next;
725 if (PREDICT_FALSE (elt_index == ~0))
729 elt = pool_elt_at_index (tsm->list_pool, elt_index);
730 ses_index = elt->value;
733 while (ses_index != ~0)
735 s = pool_elt_at_index (tsm->sessions, ses_index);
736 elt_index = elt->next;
737 elt = pool_elt_at_index (tsm->list_pool, elt_index);
738 ses_index = elt->value;
740 if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
742 new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
744 make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
745 ip->protocol, outside_fib_index, 0, 0);
746 if (clib_bihash_search_16_8
747 (&tsm->out2in_ed, &s_kv, &s_value))
754 for (i = 0; i < vec_len (sm->addresses); i++)
756 make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
757 ip->protocol, outside_fib_index, 0, 0);
758 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
760 new_addr = ip->src_address.as_u32 =
761 sm->addresses[i].addr.as_u32;
769 s = nat_ed_session_alloc (sm, u, thread_index, now);
772 nat44_delete_user_with_no_session (sm, u, thread_index);
773 nat_elog_warn ("create NAT session failed");
777 s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
778 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
779 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
780 s->out2in.addr.as_u32 = new_addr;
781 s->out2in.fib_index = outside_fib_index;
782 s->in2out.addr.as_u32 = old_addr;
783 s->in2out.fib_index = rx_fib_index;
784 s->in2out.port = s->out2in.port = ip->protocol;
786 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
787 user_session_increment (sm, u, is_sm);
789 /* Add to lookup tables */
790 make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
792 s_kv.value = s - tsm->sessions;
793 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
794 nat_elog_notice ("in2out key add failed");
796 make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
797 outside_fib_index, 0, 0);
798 s_kv.value = s - tsm->sessions;
799 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
800 nat_elog_notice ("out2in key add failed");
803 /* Update IP checksum */
805 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
806 ip->checksum = ip_csum_fold (sum);
809 nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b),
811 /* Per-user LRU list maintenance */
812 nat44_session_update_lru (sm, s, thread_index);
815 if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
816 nat44_ed_hairpinning_unknown_proto (sm, b, ip);
818 if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
819 vnet_buffer (b)->sw_if_index[VLIB_TX] = outside_fib_index;
825 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
826 vlib_node_runtime_t * node,
827 vlib_frame_t * frame, int is_slow_path,
828 int is_output_feature)
830 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
831 nat_next_t next_index;
832 snat_main_t *sm = &snat_main;
833 f64 now = vlib_time_now (vm);
834 u32 thread_index = vm->thread_index;
835 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
836 u32 tcp_packets = 0, udp_packets = 0, icmp_packets = 0, other_packets =
837 0, fragments = 0, def_slow, def_reass;
839 def_slow = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH :
840 NAT_NEXT_IN2OUT_ED_SLOW_PATH;
842 def_reass = is_output_feature ? NAT_NEXT_IN2OUT_ED_OUTPUT_REASS :
843 NAT_NEXT_IN2OUT_ED_REASS;
845 stats_node_index = is_slow_path ? sm->ed_in2out_slowpath_node_index :
846 sm->ed_in2out_node_index;
848 from = vlib_frame_vector_args (frame);
849 n_left_from = frame->n_vectors;
850 next_index = node->cached_next_index;
852 while (n_left_from > 0)
856 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
858 while (n_left_from >= 4 && n_left_to_next >= 2)
861 vlib_buffer_t *b0, *b1;
862 u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
863 new_addr0, old_addr0;
864 u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
865 new_addr1, old_addr1;
866 u16 old_port0, new_port0, old_port1, new_port1;
867 ip4_header_t *ip0, *ip1;
868 udp_header_t *udp0, *udp1;
869 tcp_header_t *tcp0, *tcp1;
870 icmp46_header_t *icmp0, *icmp1;
871 snat_session_t *s0 = 0, *s1 = 0;
872 clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
873 ip_csum_t sum0, sum1;
875 /* Prefetch next iteration. */
877 vlib_buffer_t *p2, *p3;
879 p2 = vlib_get_buffer (vm, from[2]);
880 p3 = vlib_get_buffer (vm, from[3]);
882 vlib_prefetch_buffer_header (p2, LOAD);
883 vlib_prefetch_buffer_header (p3, LOAD);
885 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
886 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
889 /* speculatively enqueue b0 and b1 to the current next frame */
890 to_next[0] = bi0 = from[0];
891 to_next[1] = bi1 = from[1];
897 b0 = vlib_get_buffer (vm, bi0);
898 b1 = vlib_get_buffer (vm, bi1);
900 if (is_output_feature)
902 // output feature fast path is enabled on the arc
903 // we need new arc_next feature
904 if (PREDICT_TRUE (!is_slow_path))
906 vnet_feature_next (&nat_buffer_opaque (b0)->arc_next, b0);
907 vnet_feature_next (&nat_buffer_opaque (b1)->arc_next, b1);
910 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
911 iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
914 next0 = nat_buffer_opaque (b0)->arc_next;
915 next1 = nat_buffer_opaque (b1)->arc_next;
917 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
920 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
922 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
925 if (PREDICT_FALSE (ip0->ttl == 1))
927 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
928 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
929 ICMP4_time_exceeded_ttl_exceeded_in_transit,
931 next0 = NAT_NEXT_ICMP_ERROR;
935 udp0 = ip4_next_header (ip0);
936 tcp0 = (tcp_header_t *) udp0;
937 icmp0 = (icmp46_header_t *) udp0;
938 proto0 = ip_proto_to_snat_proto (ip0->protocol);
942 if (PREDICT_FALSE (proto0 == ~0))
944 s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
946 thread_index, now, vm,
949 next0 = NAT_NEXT_DROP;
954 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
956 next0 = icmp_in2out_ed_slow_path
957 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
958 next0, now, thread_index, &s0);
965 if (PREDICT_FALSE (proto0 == ~0))
971 if (ip4_is_fragment (ip0))
978 if (is_output_feature)
981 (nat_not_translate_output_feature_fwd
982 (sm, ip0, thread_index, now, vm, b0)))
986 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
993 make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
994 ip0->protocol, rx_fib_index0, udp0->src_port,
997 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1001 if (is_output_feature)
1004 (nat44_ed_not_translate_output_feature
1005 (sm, ip0, ip0->protocol, udp0->src_port,
1006 udp0->dst_port, thread_index, sw_if_index0,
1007 vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1011 * Send DHCP packets to the ipv4 stack, or we won't
1012 * be able to use dhcp client on the outside interface
1015 ((b0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)
1016 && proto0 == SNAT_PROTOCOL_UDP
1017 && (udp0->dst_port ==
1018 clib_host_to_net_u16
1019 (UDP_DST_PORT_dhcp_to_server))))
1024 if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1033 slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1034 next0, thread_index, now, tcp0);
1036 if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1039 if (PREDICT_FALSE (!s0))
1050 s0 = pool_elt_at_index (tsm->sessions, value0.value);
1053 b0->flags |= VNET_BUFFER_F_IS_NATED;
1055 if (!is_output_feature)
1056 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1058 old_addr0 = ip0->src_address.as_u32;
1059 new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1060 sum0 = ip0->checksum;
1061 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1063 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1064 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1065 s0->ext_host_addr.as_u32, ip4_header_t,
1067 ip0->checksum = ip_csum_fold (sum0);
1069 old_port0 = udp0->src_port;
1070 new_port0 = udp0->src_port = s0->out2in.port;
1072 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1074 sum0 = tcp0->checksum;
1075 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1077 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1079 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1081 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1082 s0->ext_host_addr.as_u32,
1083 ip4_header_t, dst_address);
1084 sum0 = ip_csum_update (sum0, tcp0->dst_port,
1085 s0->ext_host_port, ip4_header_t,
1087 tcp0->dst_port = s0->ext_host_port;
1088 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1090 mss_clamping (sm, tcp0, &sum0);
1091 tcp0->checksum = ip_csum_fold (sum0);
1093 if (nat44_set_tcp_session_state_i2o
1094 (sm, s0, tcp0, thread_index))
1097 else if (udp0->checksum)
1099 sum0 = udp0->checksum;
1100 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1102 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1104 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1106 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1107 s0->ext_host_addr.as_u32,
1108 ip4_header_t, dst_address);
1109 sum0 = ip_csum_update (sum0, tcp0->dst_port,
1110 s0->ext_host_port, ip4_header_t,
1112 udp0->dst_port = s0->ext_host_port;
1113 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1115 udp0->checksum = ip_csum_fold (sum0);
1120 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1122 udp0->dst_port = s0->ext_host_port;
1123 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1129 nat44_session_update_counters (s0, now,
1130 vlib_buffer_length_in_chain (vm,
1133 /* Per-user LRU list maintenance */
1134 nat44_session_update_lru (sm, s0, thread_index);
1137 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1138 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1140 nat_in2out_ed_trace_t *t =
1141 vlib_add_trace (vm, node, b0, sizeof (*t));
1142 t->is_slow_path = is_slow_path;
1143 t->sw_if_index = sw_if_index0;
1144 t->next_index = next0;
1145 t->session_index = ~0;
1147 t->session_index = s0 - tsm->sessions;
1150 pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
1152 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1155 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1157 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1160 if (PREDICT_FALSE (ip1->ttl == 1))
1162 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1163 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1164 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1166 next1 = NAT_NEXT_ICMP_ERROR;
1170 udp1 = ip4_next_header (ip1);
1171 tcp1 = (tcp_header_t *) udp1;
1172 icmp1 = (icmp46_header_t *) udp1;
1173 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1177 if (PREDICT_FALSE (proto1 == ~0))
1179 s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
1181 thread_index, now, vm,
1184 next1 = NAT_NEXT_DROP;
1189 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1191 next1 = icmp_in2out_ed_slow_path
1192 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1193 next1, now, thread_index, &s1);
1200 if (PREDICT_FALSE (proto1 == ~0))
1206 if (ip4_is_fragment (ip1))
1213 if (is_output_feature)
1216 (nat_not_translate_output_feature_fwd
1217 (sm, ip1, thread_index, now, vm, b1)))
1221 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1228 make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address,
1229 ip1->protocol, rx_fib_index1, udp1->src_port,
1232 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
1236 if (is_output_feature)
1239 (nat44_ed_not_translate_output_feature
1240 (sm, ip1, ip1->protocol, udp1->src_port,
1241 udp1->dst_port, thread_index, sw_if_index1,
1242 vnet_buffer (b1)->sw_if_index[VLIB_TX])))
1246 * Send DHCP packets to the ipv4 stack, or we won't
1247 * be able to use dhcp client on the outside interface
1250 ((b1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)
1251 && proto1 == SNAT_PROTOCOL_UDP
1252 && (udp1->dst_port ==
1253 clib_host_to_net_u16
1254 (UDP_DST_PORT_dhcp_to_server))))
1259 if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1268 slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
1269 next1, thread_index, now, tcp1);
1271 if (PREDICT_FALSE (next1 == NAT_NEXT_DROP))
1274 if (PREDICT_FALSE (!s1))
1285 s1 = pool_elt_at_index (tsm->sessions, value1.value);
1288 b1->flags |= VNET_BUFFER_F_IS_NATED;
1290 if (!is_output_feature)
1291 vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1293 old_addr1 = ip1->src_address.as_u32;
1294 new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
1295 sum1 = ip1->checksum;
1296 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1298 if (PREDICT_FALSE (is_twice_nat_session (s1)))
1299 sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1300 s1->ext_host_addr.as_u32, ip4_header_t,
1302 ip1->checksum = ip_csum_fold (sum1);
1304 old_port1 = udp1->src_port;
1305 new_port1 = udp1->src_port = s1->out2in.port;
1307 if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
1309 sum1 = tcp1->checksum;
1310 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1312 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1314 if (PREDICT_FALSE (is_twice_nat_session (s1)))
1316 sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1317 s1->ext_host_addr.as_u32,
1318 ip4_header_t, dst_address);
1319 sum1 = ip_csum_update (sum1, tcp1->dst_port,
1320 s1->ext_host_port, ip4_header_t,
1322 tcp1->dst_port = s1->ext_host_port;
1323 ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1325 tcp1->checksum = ip_csum_fold (sum1);
1326 mss_clamping (sm, tcp1, &sum1);
1328 if (nat44_set_tcp_session_state_i2o
1329 (sm, s1, tcp1, thread_index))
1332 else if (udp1->checksum)
1334 sum1 = udp1->checksum;
1335 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
1337 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
1340 if (PREDICT_FALSE (is_twice_nat_session (s1)))
1342 sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
1343 s1->ext_host_addr.as_u32,
1344 ip4_header_t, dst_address);
1345 sum1 = ip_csum_update (sum1, tcp1->dst_port,
1346 s1->ext_host_port, ip4_header_t,
1348 udp1->dst_port = s1->ext_host_port;
1349 ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1351 udp1->checksum = ip_csum_fold (sum1);
1356 if (PREDICT_FALSE (is_twice_nat_session (s1)))
1358 udp1->dst_port = s1->ext_host_port;
1359 ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
1365 nat44_session_update_counters (s1, now,
1366 vlib_buffer_length_in_chain (vm, b1),
1368 /* Per-user LRU list maintenance */
1369 nat44_session_update_lru (sm, s1, thread_index);
1372 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1373 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1375 nat_in2out_ed_trace_t *t =
1376 vlib_add_trace (vm, node, b1, sizeof (*t));
1377 t->is_slow_path = is_slow_path;
1378 t->sw_if_index = sw_if_index1;
1379 t->next_index = next1;
1380 t->session_index = ~0;
1382 t->session_index = s1 - tsm->sessions;
1385 pkts_processed += next1 == nat_buffer_opaque (b1)->arc_next;
1388 /* verify speculative enqueues, maybe switch current next frame */
1389 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1390 to_next, n_left_to_next,
1391 bi0, bi1, next0, next1);
1394 while (n_left_from > 0 && n_left_to_next > 0)
1398 u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
1399 new_addr0, old_addr0;
1400 u16 old_port0, new_port0;
1404 icmp46_header_t *icmp0;
1405 snat_session_t *s0 = 0;
1406 clib_bihash_kv_16_8_t kv0, value0;
1409 /* speculatively enqueue b0 to the current next frame */
1415 n_left_to_next -= 1;
1417 b0 = vlib_get_buffer (vm, bi0);
1419 if (is_output_feature)
1421 // output feature fast path is enabled on the arc
1422 // we need new arc_next feature
1423 if (PREDICT_TRUE (!is_slow_path))
1424 vnet_feature_next (&nat_buffer_opaque (b0)->arc_next, b0);
1426 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1429 next0 = nat_buffer_opaque (b0)->arc_next;
1431 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1434 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1436 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1439 if (PREDICT_FALSE (ip0->ttl == 1))
1441 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1442 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1443 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1445 next0 = NAT_NEXT_ICMP_ERROR;
1449 udp0 = ip4_next_header (ip0);
1450 tcp0 = (tcp_header_t *) udp0;
1451 icmp0 = (icmp46_header_t *) udp0;
1452 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1456 if (PREDICT_FALSE (proto0 == ~0))
1458 s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
1460 thread_index, now, vm,
1463 next0 = NAT_NEXT_DROP;
1468 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1470 next0 = icmp_in2out_ed_slow_path
1471 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1472 next0, now, thread_index, &s0);
1479 if (PREDICT_FALSE (proto0 == ~0))
1485 if (ip4_is_fragment (ip0))
1492 if (is_output_feature)
1495 (nat_not_translate_output_feature_fwd
1496 (sm, ip0, thread_index, now, vm, b0)))
1500 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1507 make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1508 ip0->protocol, rx_fib_index0, udp0->src_port,
1511 if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
1515 if (is_output_feature)
1518 (nat44_ed_not_translate_output_feature
1519 (sm, ip0, ip0->protocol, udp0->src_port,
1520 udp0->dst_port, thread_index, sw_if_index0,
1521 vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1525 * Send DHCP packets to the ipv4 stack, or we won't
1526 * be able to use dhcp client on the outside interface
1529 ((b0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)
1530 && proto0 == SNAT_PROTOCOL_UDP
1531 && (udp0->dst_port ==
1532 clib_host_to_net_u16
1533 (UDP_DST_PORT_dhcp_to_server))))
1538 if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1547 slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
1548 next0, thread_index, now, tcp0);
1550 if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1553 if (PREDICT_FALSE (!s0))
1564 s0 = pool_elt_at_index (tsm->sessions, value0.value);
1567 b0->flags |= VNET_BUFFER_F_IS_NATED;
1569 if (!is_output_feature)
1570 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1572 old_addr0 = ip0->src_address.as_u32;
1573 new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
1574 sum0 = ip0->checksum;
1575 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1577 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1578 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1579 s0->ext_host_addr.as_u32, ip4_header_t,
1581 ip0->checksum = ip_csum_fold (sum0);
1583 old_port0 = udp0->src_port;
1584 new_port0 = udp0->src_port = s0->out2in.port;
1586 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1588 sum0 = tcp0->checksum;
1589 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1591 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1593 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1595 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1596 s0->ext_host_addr.as_u32,
1597 ip4_header_t, dst_address);
1598 sum0 = ip_csum_update (sum0, tcp0->dst_port,
1599 s0->ext_host_port, ip4_header_t,
1601 tcp0->dst_port = s0->ext_host_port;
1602 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1604 mss_clamping (sm, tcp0, &sum0);
1605 tcp0->checksum = ip_csum_fold (sum0);
1607 if (nat44_set_tcp_session_state_i2o
1608 (sm, s0, tcp0, thread_index))
1611 else if (udp0->checksum)
1613 sum0 = udp0->checksum;
1614 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1616 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1618 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1620 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1621 s0->ext_host_addr.as_u32,
1622 ip4_header_t, dst_address);
1623 sum0 = ip_csum_update (sum0, tcp0->dst_port,
1624 s0->ext_host_port, ip4_header_t,
1626 udp0->dst_port = s0->ext_host_port;
1627 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1629 udp0->checksum = ip_csum_fold (sum0);
1634 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1636 udp0->dst_port = s0->ext_host_port;
1637 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1643 nat44_session_update_counters (s0, now,
1644 vlib_buffer_length_in_chain (vm, b0),
1646 /* Per-user LRU list maintenance */
1647 nat44_session_update_lru (sm, s0, thread_index);
1650 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1651 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1653 nat_in2out_ed_trace_t *t =
1654 vlib_add_trace (vm, node, b0, sizeof (*t));
1655 t->is_slow_path = is_slow_path;
1656 t->sw_if_index = sw_if_index0;
1657 t->next_index = next0;
1658 t->session_index = ~0;
1660 t->session_index = s0 - tsm->sessions;
1663 pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
1665 /* verify speculative enqueue, maybe switch current next frame */
1666 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1667 to_next, n_left_to_next,
1671 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1674 vlib_node_increment_counter (vm, stats_node_index,
1675 NAT_IN2OUT_ED_ERROR_IN2OUT_PACKETS,
1677 vlib_node_increment_counter (vm, stats_node_index,
1678 NAT_IN2OUT_ED_ERROR_TCP_PACKETS, tcp_packets);
1679 vlib_node_increment_counter (vm, stats_node_index,
1680 NAT_IN2OUT_ED_ERROR_UDP_PACKETS, udp_packets);
1681 vlib_node_increment_counter (vm, stats_node_index,
1682 NAT_IN2OUT_ED_ERROR_ICMP_PACKETS,
1684 vlib_node_increment_counter (vm, stats_node_index,
1685 NAT_IN2OUT_ED_ERROR_OTHER_PACKETS,
1687 vlib_node_increment_counter (vm, stats_node_index,
1688 NAT_IN2OUT_ED_ERROR_FRAGMENTS, fragments);
1690 return frame->n_vectors;
1694 nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm,
1695 vlib_node_runtime_t * node,
1696 vlib_frame_t * frame,
1697 int is_output_feature)
1699 u32 n_left_from, *from, *to_next;
1700 nat_next_t next_index;
1701 u32 pkts_processed = 0, cached_fragments = 0;
1702 snat_main_t *sm = &snat_main;
1703 f64 now = vlib_time_now (vm);
1704 u32 thread_index = vm->thread_index;
1705 snat_main_per_thread_data_t *per_thread_data =
1706 &sm->per_thread_data[thread_index];
1707 u32 *fragments_to_drop = 0;
1708 u32 *fragments_to_loopback = 0;
1710 from = vlib_frame_vector_args (frame);
1711 n_left_from = frame->n_vectors;
1712 next_index = node->cached_next_index;
1714 while (n_left_from > 0)
1718 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1720 while (n_left_from > 0 && n_left_to_next > 0)
1722 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1723 u32 iph_offset0 = 0;
1727 ip4_header_t *ip0 = 0;
1728 nat_reass_ip4_t *reass0;
1731 icmp46_header_t *icmp0;
1732 clib_bihash_kv_16_8_t kv0, value0;
1733 snat_session_t *s0 = 0;
1734 u16 old_port0, new_port0;
1737 /* speculatively enqueue b0 to the current next frame */
1743 n_left_to_next -= 1;
1745 b0 = vlib_get_buffer (vm, bi0);
1747 next0 = nat_buffer_opaque (b0)->arc_next;
1749 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1751 fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1754 if (PREDICT_FALSE (nat_reass_is_drop_frag (0)))
1756 next0 = NAT_NEXT_DROP;
1757 b0->error = node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT];
1761 if (is_output_feature)
1762 iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1764 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1767 udp0 = ip4_next_header (ip0);
1768 tcp0 = (tcp_header_t *) udp0;
1769 icmp0 = (icmp46_header_t *) udp0;
1770 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1772 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1776 1, &fragments_to_drop);
1778 if (PREDICT_FALSE (!reass0))
1780 next0 = NAT_NEXT_DROP;
1781 b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_REASS];
1782 nat_elog_notice ("maximum reassemblies exceeded");
1786 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1788 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1790 if (is_output_feature)
1793 (nat_not_translate_output_feature_fwd
1794 (sm, ip0, thread_index, now, vm, b0)))
1795 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1799 next0 = icmp_in2out_ed_slow_path
1800 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1801 next0, now, thread_index, &s0);
1803 if (PREDICT_TRUE (next0 != NAT_NEXT_DROP))
1806 reass0->sess_index = s0 - per_thread_data->sessions;
1808 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1809 nat_ip4_reass_get_frags (reass0,
1810 &fragments_to_loopback);
1816 make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address,
1817 ip0->protocol, rx_fib_index0, udp0->src_port,
1820 if (clib_bihash_search_16_8
1821 (&per_thread_data->in2out_ed, &kv0, &value0))
1823 if (is_output_feature)
1826 (nat44_ed_not_translate_output_feature
1827 (sm, ip0, ip0->protocol, udp0->src_port,
1828 udp0->dst_port, thread_index, sw_if_index0,
1829 vnet_buffer (b0)->sw_if_index[VLIB_TX])))
1831 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1832 nat_ip4_reass_get_frags (reass0,
1833 &fragments_to_loopback);
1838 * Send DHCP packets to the ipv4 stack, or we won't
1839 * be able to use dhcp client on the outside interface
1842 ((b0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)
1843 && proto0 == SNAT_PROTOCOL_UDP
1844 && (udp0->dst_port ==
1845 clib_host_to_net_u16
1846 (UDP_DST_PORT_dhcp_to_server))))
1851 if (PREDICT_FALSE (nat44_ed_not_translate (sm, node,
1857 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1858 nat_ip4_reass_get_frags (reass0,
1859 &fragments_to_loopback);
1864 next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0,
1865 &s0, node, next0, thread_index, now,
1868 if (PREDICT_FALSE (next0 == NAT_NEXT_DROP))
1871 if (PREDICT_FALSE (!s0))
1873 reass0->flags |= NAT_REASS_FLAG_ED_DONT_TRANSLATE;
1877 reass0->sess_index = s0 - per_thread_data->sessions;
1881 s0 = pool_elt_at_index (per_thread_data->sessions,
1883 reass0->sess_index = value0.value;
1885 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1889 if (reass0->flags & NAT_REASS_FLAG_ED_DONT_TRANSLATE)
1891 if (PREDICT_FALSE (reass0->sess_index == (u32) ~ 0))
1893 if (nat_ip4_reass_add_fragment
1894 (thread_index, reass0, bi0, &fragments_to_drop))
1896 b0->error = node->errors[NAT_IN2OUT_ED_ERROR_MAX_FRAG];
1898 ("maximum fragments per reassembly exceeded");
1899 next0 = NAT_NEXT_DROP;
1905 s0 = pool_elt_at_index (per_thread_data->sessions,
1906 reass0->sess_index);
1909 old_addr0 = ip0->src_address.as_u32;
1910 ip0->src_address = s0->out2in.addr;
1911 new_addr0 = ip0->src_address.as_u32;
1912 if (!is_output_feature)
1913 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1915 sum0 = ip0->checksum;
1916 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1918 src_address /* changed member */ );
1919 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1920 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1921 s0->ext_host_addr.as_u32, ip4_header_t,
1923 ip0->checksum = ip_csum_fold (sum0);
1925 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1927 old_port0 = udp0->src_port;
1928 new_port0 = udp0->src_port = s0->out2in.port;
1930 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
1932 sum0 = tcp0->checksum;
1933 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1935 dst_address /* changed member */ );
1936 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1937 ip4_header_t /* cheat */ ,
1938 length /* changed member */ );
1939 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1941 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1942 s0->ext_host_addr.as_u32,
1943 ip4_header_t, dst_address);
1944 sum0 = ip_csum_update (sum0, tcp0->dst_port,
1945 s0->ext_host_port, ip4_header_t,
1947 tcp0->dst_port = s0->ext_host_port;
1948 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1950 tcp0->checksum = ip_csum_fold (sum0);
1952 else if (udp0->checksum)
1954 sum0 = udp0->checksum;
1956 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1959 ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
1961 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1963 sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
1964 s0->ext_host_addr.as_u32,
1965 ip4_header_t, dst_address);
1966 sum0 = ip_csum_update (sum0, tcp0->dst_port,
1967 s0->ext_host_port, ip4_header_t,
1969 udp0->dst_port = s0->ext_host_port;
1970 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1972 udp0->checksum = ip_csum_fold (sum0);
1976 if (PREDICT_FALSE (is_twice_nat_session (s0)))
1978 udp0->dst_port = s0->ext_host_port;
1979 ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
1985 nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
1986 s0->ext_host_port, proto0, 1);
1989 nat44_session_update_counters (s0, now,
1990 vlib_buffer_length_in_chain (vm, b0),
1992 /* Per-user LRU list maintenance */
1993 nat44_session_update_lru (sm, s0, thread_index);
1996 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1997 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1999 nat44_reass_trace_t *t =
2000 vlib_add_trace (vm, node, b0, sizeof (*t));
2001 t->cached = cached0;
2002 t->sw_if_index = sw_if_index0;
2003 t->next_index = next0;
2014 pkts_processed += next0 == nat_buffer_opaque (b0)->arc_next;
2016 /* verify speculative enqueue, maybe switch current next frame */
2017 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2018 to_next, n_left_to_next,
2022 if (n_left_from == 0 && vec_len (fragments_to_loopback))
2024 from = vlib_frame_vector_args (frame);
2025 u32 len = vec_len (fragments_to_loopback);
2026 if (len <= VLIB_FRAME_SIZE)
2028 clib_memcpy_fast (from, fragments_to_loopback,
2029 sizeof (u32) * len);
2031 vec_reset_length (fragments_to_loopback);
2035 clib_memcpy_fast (from, fragments_to_loopback +
2036 (len - VLIB_FRAME_SIZE),
2037 sizeof (u32) * VLIB_FRAME_SIZE);
2038 n_left_from = VLIB_FRAME_SIZE;
2039 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2044 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2047 vlib_node_increment_counter (vm, sm->ed_in2out_reass_node_index,
2048 NAT_IN2OUT_ED_ERROR_PROCESSED_FRAGMENTS,
2050 vlib_node_increment_counter (vm, sm->ed_in2out_reass_node_index,
2051 NAT_IN2OUT_ED_ERROR_CACHED_FRAGMENTS,
2054 nat_send_all_to_node (vm, fragments_to_drop, node,
2055 &node->errors[NAT_IN2OUT_ED_ERROR_DROP_FRAGMENT],
2058 vec_free (fragments_to_drop);
2059 vec_free (fragments_to_loopback);
2060 return frame->n_vectors;
2063 VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
2064 vlib_node_runtime_t * node,
2065 vlib_frame_t * frame)
2067 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
2071 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
2072 .name = "nat44-ed-in2out",
2073 .vector_size = sizeof (u32),
2074 .sibling_of = "nat-default",
2075 .format_trace = format_nat_in2out_ed_trace,
2076 .type = VLIB_NODE_TYPE_INTERNAL,
2077 .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2078 .error_strings = nat_in2out_ed_error_strings,
2079 .runtime_data_bytes = sizeof (snat_runtime_t),
2083 VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
2084 vlib_node_runtime_t * node,
2085 vlib_frame_t * frame)
2087 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
2091 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
2092 .name = "nat44-ed-in2out-output",
2093 .vector_size = sizeof (u32),
2094 .sibling_of = "nat-default",
2095 .format_trace = format_nat_in2out_ed_trace,
2096 .type = VLIB_NODE_TYPE_INTERNAL,
2097 .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2098 .error_strings = nat_in2out_ed_error_strings,
2099 .runtime_data_bytes = sizeof (snat_runtime_t),
2103 VLIB_NODE_FN (nat44_ed_in2out_slowpath_node) (vlib_main_t * vm,
2104 vlib_node_runtime_t * node,
2105 vlib_frame_t * frame)
2107 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
2111 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
2112 .name = "nat44-ed-in2out-slowpath",
2113 .vector_size = sizeof (u32),
2114 .sibling_of = "nat-default",
2115 .format_trace = format_nat_in2out_ed_trace,
2116 .type = VLIB_NODE_TYPE_INTERNAL,
2117 .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2118 .error_strings = nat_in2out_ed_error_strings,
2119 .runtime_data_bytes = sizeof (snat_runtime_t),
2123 VLIB_NODE_FN (nat44_ed_in2out_output_slowpath_node) (vlib_main_t * vm,
2124 vlib_node_runtime_t *
2126 vlib_frame_t * frame)
2128 return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
2132 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
2133 .name = "nat44-ed-in2out-output-slowpath",
2134 .vector_size = sizeof (u32),
2135 .sibling_of = "nat-default",
2136 .format_trace = format_nat_in2out_ed_trace,
2137 .type = VLIB_NODE_TYPE_INTERNAL,
2138 .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2139 .error_strings = nat_in2out_ed_error_strings,
2140 .runtime_data_bytes = sizeof (snat_runtime_t),
2145 VLIB_NODE_FN (nat44_ed_in2out_reass_node) (vlib_main_t * vm,
2146 vlib_node_runtime_t * node,
2147 vlib_frame_t * frame)
2149 return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 0);
2153 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_node) = {
2154 .name = "nat44-ed-in2out-reass",
2155 .vector_size = sizeof (u32),
2156 .sibling_of = "nat-default",
2157 .format_trace = format_nat44_reass_trace,
2158 .type = VLIB_NODE_TYPE_INTERNAL,
2159 .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2160 .error_strings = nat_in2out_ed_error_strings,
2164 VLIB_NODE_FN (nat44_ed_in2out_reass_output_node) (vlib_main_t * vm,
2165 vlib_node_runtime_t * node,
2166 vlib_frame_t * frame)
2168 return nat44_ed_in2out_reass_node_fn_inline (vm, node, frame, 1);
2172 VLIB_REGISTER_NODE (nat44_ed_in2out_reass_output_node) = {
2173 .name = "nat44-ed-in2out-reass-output",
2174 .vector_size = sizeof (u32),
2175 .sibling_of = "nat-default",
2176 .format_trace = format_nat44_reass_trace,
2177 .type = VLIB_NODE_TYPE_INTERNAL,
2178 .n_errors = ARRAY_LEN (nat_in2out_ed_error_strings),
2179 .error_strings = nat_in2out_ed_error_strings,
2184 format_nat_pre_trace (u8 * s, va_list * args)
2186 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2187 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2188 nat_pre_trace_t *t = va_arg (*args, nat_pre_trace_t *);
2189 return format (s, "in2out next_index %d", t->next_index);
2192 VLIB_NODE_FN (nat_pre_in2out_node) (vlib_main_t * vm,
2193 vlib_node_runtime_t * node,
2194 vlib_frame_t * frame)
2196 return nat_pre_node_fn_inline (vm, node, frame,
2197 NAT_NEXT_IN2OUT_ED_FAST_PATH);
2201 VLIB_REGISTER_NODE (nat_pre_in2out_node) = {
2202 .name = "nat-pre-in2out",
2203 .vector_size = sizeof (u32),
2204 .sibling_of = "nat-default",
2205 .format_trace = format_nat_pre_trace,
2206 .type = VLIB_NODE_TYPE_INTERNAL,
2212 * fd.io coding-style-patch-verification: ON
2215 * eval: (c-set-style "gnu")