2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
26 #include <nat/nat_ipfix_logging.h>
27 #include <nat/nat_det.h>
28 #include <nat/nat_reass.h>
29 #include <nat/nat_inlines.h>
31 #include <vppinfra/hash.h>
32 #include <vppinfra/error.h>
33 #include <vppinfra/elog.h>
39 } snat_out2in_trace_t;
42 u32 next_worker_index;
44 } snat_out2in_worker_handoff_trace_t;
46 /* packet trace format function */
47 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
49 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
50 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
51 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
53 s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
54 t->sw_if_index, t->next_index, t->session_index);
58 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
60 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
61 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
62 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
64 s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
65 t->sw_if_index, t->next_index);
69 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
71 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
72 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
73 snat_out2in_worker_handoff_trace_t * t =
74 va_arg (*args, snat_out2in_worker_handoff_trace_t *);
77 m = t->do_handoff ? "next worker" : "same worker";
78 s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
87 } nat44_out2in_reass_trace_t;
89 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
91 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
92 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
93 nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
95 s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
96 t->sw_if_index, t->next_index,
97 t->cached ? "cached" : "translated");
102 vlib_node_registration_t snat_out2in_node;
103 vlib_node_registration_t snat_out2in_fast_node;
104 vlib_node_registration_t snat_out2in_worker_handoff_node;
105 vlib_node_registration_t snat_det_out2in_node;
106 vlib_node_registration_t nat44_out2in_reass_node;
108 #define foreach_snat_out2in_error \
109 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
110 _(OUT2IN_PACKETS, "Good out2in packets processed") \
111 _(OUT_OF_PORTS, "Out of ports") \
112 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
113 _(NO_TRANSLATION, "No translation") \
114 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
115 _(DROP_FRAGMENT, "Drop fragment") \
116 _(MAX_REASS, "Maximum reassemblies exceeded") \
117 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
118 _(FQ_CONGESTED, "Handoff frame queue congested")
121 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
122 foreach_snat_out2in_error
125 } snat_out2in_error_t;
127 static char * snat_out2in_error_strings[] = {
128 #define _(sym,string) string,
129 foreach_snat_out2in_error
134 SNAT_OUT2IN_NEXT_DROP,
135 SNAT_OUT2IN_NEXT_LOOKUP,
136 SNAT_OUT2IN_NEXT_ICMP_ERROR,
137 SNAT_OUT2IN_NEXT_REASS,
138 SNAT_OUT2IN_NEXT_IN2OUT,
140 } snat_out2in_next_t;
143 * @brief Create session for static mapping.
145 * Create NAT session initiated by host from external network with static
148 * @param sm NAT main.
149 * @param b0 Vlib buffer.
150 * @param in2out In2out NAT44 session key.
151 * @param out2in Out2in NAT44 session key.
152 * @param node Vlib node.
154 * @returns SNAT session if successfully created otherwise 0.
156 static inline snat_session_t *
157 create_session_for_static_mapping (snat_main_t *sm,
159 snat_session_key_t in2out,
160 snat_session_key_t out2in,
161 vlib_node_runtime_t * node,
166 clib_bihash_kv_8_8_t kv0;
170 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
172 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
173 nat_log_notice ("maximum sessions exceeded");
177 ip0 = vlib_buffer_get_current (b0);
178 udp0 = ip4_next_header (ip0);
180 u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
183 nat_log_warn ("create NAT user failed");
187 s = nat_session_alloc_or_recycle (sm, u, thread_index);
190 nat_log_warn ("create NAT session failed");
194 s->outside_address_index = ~0;
195 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
196 s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
197 s->ext_host_port = udp0->src_port;
198 user_session_increment (sm, u, 1 /* static */);
201 s->in2out.protocol = out2in.protocol;
203 /* Add to translation hashes */
204 kv0.key = s->in2out.as_u64;
205 kv0.value = s - sm->per_thread_data[thread_index].sessions;
206 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
208 nat_log_notice ("in2out key add failed");
210 kv0.key = s->out2in.as_u64;
212 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
214 nat_log_notice ("out2in key add failed");
217 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
218 s->out2in.addr.as_u32,
222 s->in2out.fib_index);
227 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
228 snat_session_key_t *p_key0)
230 icmp46_header_t *icmp0;
231 snat_session_key_t key0;
232 icmp_echo_header_t *echo0, *inner_echo0 = 0;
233 ip4_header_t *inner_ip0;
235 icmp46_header_t *inner_icmp0;
237 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
238 echo0 = (icmp_echo_header_t *)(icmp0+1);
240 if (!icmp_is_error_message (icmp0))
242 key0.protocol = SNAT_PROTOCOL_ICMP;
243 key0.addr = ip0->dst_address;
244 key0.port = echo0->identifier;
248 inner_ip0 = (ip4_header_t *)(echo0+1);
249 l4_header = ip4_next_header (inner_ip0);
250 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
251 key0.addr = inner_ip0->src_address;
252 switch (key0.protocol)
254 case SNAT_PROTOCOL_ICMP:
255 inner_icmp0 = (icmp46_header_t*)l4_header;
256 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
257 key0.port = inner_echo0->identifier;
259 case SNAT_PROTOCOL_UDP:
260 case SNAT_PROTOCOL_TCP:
261 key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
264 return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
268 return -1; /* success */
271 static_always_inline int
272 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
274 icmp46_header_t *icmp0;
275 nat_ed_ses_key_t key0;
276 icmp_echo_header_t *echo0, *inner_echo0 = 0;
277 ip4_header_t *inner_ip0;
279 icmp46_header_t *inner_icmp0;
281 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
282 echo0 = (icmp_echo_header_t *)(icmp0+1);
284 if (!icmp_is_error_message (icmp0))
286 key0.proto = IP_PROTOCOL_ICMP;
287 key0.l_addr = ip0->dst_address;
288 key0.r_addr = ip0->src_address;
289 key0.l_port = key0.r_port = echo0->identifier;
293 inner_ip0 = (ip4_header_t *)(echo0+1);
294 l4_header = ip4_next_header (inner_ip0);
295 key0.proto = inner_ip0->protocol;
296 key0.l_addr = inner_ip0->src_address;
297 key0.r_addr = inner_ip0->dst_address;
298 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
300 case SNAT_PROTOCOL_ICMP:
301 inner_icmp0 = (icmp46_header_t*)l4_header;
302 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
303 key0.l_port = key0.r_port = inner_echo0->identifier;
305 case SNAT_PROTOCOL_UDP:
306 case SNAT_PROTOCOL_TCP:
307 key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
308 key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
319 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
322 snat_session_key_t key;
323 clib_bihash_kv_8_8_t kv, value;
325 key.addr = ip->src_address;
327 key.protocol = proto;
328 key.fib_index = sm->inside_fib_index;
331 if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
339 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
342 nat_ed_ses_key_t key;
343 clib_bihash_kv_16_8_t kv, value;
346 snat_session_t *s = 0;
347 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
348 f64 now = vlib_time_now (sm->vlib_main);
350 if (ip->protocol == IP_PROTOCOL_ICMP)
352 if (icmp_get_ed_key (ip, &key))
355 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
357 udp = ip4_next_header(ip);
358 key.r_addr = ip->src_address;
359 key.l_addr = ip->dst_address;
360 key.proto = ip->protocol;
361 key.l_port = udp->dst_port;
362 key.r_port = udp->src_port;
366 key.r_addr = ip->src_address;
367 key.l_addr = ip->dst_address;
368 key.proto = ip->protocol;
369 key.l_port = key.r_port = 0;
372 kv.key[0] = key.as_u64[0];
373 kv.key[1] = key.as_u64[1];
375 if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
377 s = pool_elt_at_index (tsm->sessions, value.value);
381 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
384 u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index, thread_index);
387 nat_log_warn ("create NAT user failed");
391 s = nat_session_alloc_or_recycle (sm, u, thread_index);
394 nat_log_warn ("create NAT session failed");
398 s->ext_host_addr = key.r_addr;
399 s->ext_host_port = key.r_port;
400 s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
401 s->outside_address_index = ~0;
402 s->out2in.addr = key.l_addr;
403 s->out2in.port = key.l_port;
404 s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
405 s->out2in.fib_index = 0;
406 s->in2out = s->out2in;
407 user_session_increment (sm, u, 0);
409 kv.value = s - tsm->sessions;
410 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
411 nat_log_notice ("in2out_ed key add failed");
414 if (ip->protocol == IP_PROTOCOL_TCP)
416 tcp_header_t *tcp = ip4_next_header(ip);
417 if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
420 /* Per-user LRU list maintenance */
421 nat44_session_update_lru (sm, s, thread_index);
423 nat44_session_update_counters (s, now, 0);
427 * Get address and port values to be used for ICMP packet translation
428 * and create session if needed
430 * @param[in,out] sm NAT main
431 * @param[in,out] node NAT node runtime
432 * @param[in] thread_index thread index
433 * @param[in,out] b0 buffer containing packet to be translated
434 * @param[out] p_proto protocol used for matching
435 * @param[out] p_value address and port after NAT translation
436 * @param[out] p_dont_translate if packet should not be translated
437 * @param d optional parameter
438 * @param e optional parameter
440 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
441 u32 thread_index, vlib_buffer_t *b0,
442 ip4_header_t *ip0, u8 *p_proto,
443 snat_session_key_t *p_value,
444 u8 *p_dont_translate, void *d, void *e)
446 icmp46_header_t *icmp0;
449 snat_session_key_t key0;
450 snat_session_key_t sm0;
451 snat_session_t *s0 = 0;
452 u8 dont_translate = 0;
453 clib_bihash_kv_8_8_t kv0, value0;
458 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
459 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
460 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
464 err = icmp_get_key (ip0, &key0);
467 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
468 next0 = SNAT_OUT2IN_NEXT_DROP;
471 key0.fib_index = rx_fib_index0;
473 kv0.key = key0.as_u64;
475 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
478 /* Try to match static mapping by external address and port,
479 destination address and port in packet */
480 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
482 if (!sm->forwarding_enabled)
484 /* Don't NAT packet aimed at the intfc address */
485 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
486 ip0->dst_address.as_u32)))
491 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
492 next0 = SNAT_OUT2IN_NEXT_DROP;
498 if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
500 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
503 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
508 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
509 (icmp0->type != ICMP4_echo_request || !is_addr_only)))
511 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
512 next0 = SNAT_OUT2IN_NEXT_DROP;
516 /* Create session initiated by host from external network */
517 s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
522 next0 = SNAT_OUT2IN_NEXT_DROP;
528 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
529 icmp0->type != ICMP4_echo_request &&
530 !icmp_is_error_message (icmp0)))
532 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
533 next0 = SNAT_OUT2IN_NEXT_DROP;
537 if (PREDICT_FALSE (value0.value == ~0ULL))
539 nat_ed_ses_key_t key;
540 clib_bihash_kv_16_8_t s_kv, s_value;
544 if (icmp_get_ed_key (ip0, &key))
546 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
547 next0 = SNAT_OUT2IN_NEXT_DROP;
550 key.fib_index = rx_fib_index0;
551 s_kv.key[0] = key.as_u64[0];
552 s_kv.key[1] = key.as_u64[1];
553 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
554 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
558 next0 = SNAT_OUT2IN_NEXT_DROP;
563 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
568 *p_proto = key0.protocol;
570 *p_value = s0->in2out;
571 *p_dont_translate = dont_translate;
573 *(snat_session_t**)d = s0;
578 * Get address and port values to be used for ICMP packet translation
580 * @param[in] sm NAT main
581 * @param[in,out] node NAT node runtime
582 * @param[in] thread_index thread index
583 * @param[in,out] b0 buffer containing packet to be translated
584 * @param[out] p_proto protocol used for matching
585 * @param[out] p_value address and port after NAT translation
586 * @param[out] p_dont_translate if packet should not be translated
587 * @param d optional parameter
588 * @param e optional parameter
590 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
591 u32 thread_index, vlib_buffer_t *b0,
592 ip4_header_t *ip0, u8 *p_proto,
593 snat_session_key_t *p_value,
594 u8 *p_dont_translate, void *d, void *e)
596 icmp46_header_t *icmp0;
599 snat_session_key_t key0;
600 snat_session_key_t sm0;
601 u8 dont_translate = 0;
606 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
607 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
608 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
610 err = icmp_get_key (ip0, &key0);
613 b0->error = node->errors[err];
614 next0 = SNAT_OUT2IN_NEXT_DROP;
617 key0.fib_index = rx_fib_index0;
619 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
621 /* Don't NAT packet aimed at the intfc address */
622 if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
627 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
628 next0 = SNAT_OUT2IN_NEXT_DROP;
632 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
633 (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
634 !icmp_is_error_message (icmp0)))
636 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
637 next0 = SNAT_OUT2IN_NEXT_DROP;
644 *p_proto = key0.protocol;
645 *p_dont_translate = dont_translate;
649 static inline u32 icmp_out2in (snat_main_t *sm,
652 icmp46_header_t * icmp0,
655 vlib_node_runtime_t * node,
661 snat_session_key_t sm0;
663 icmp_echo_header_t *echo0, *inner_echo0 = 0;
664 ip4_header_t *inner_ip0 = 0;
666 icmp46_header_t *inner_icmp0;
668 u32 new_addr0, old_addr0;
669 u16 old_id0, new_id0;
674 echo0 = (icmp_echo_header_t *)(icmp0+1);
676 next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
677 &protocol, &sm0, &dont_translate, d, e);
680 if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
683 sum0 = ip_incremental_checksum (0, icmp0,
684 ntohs(ip0->length) - ip4_header_bytes (ip0));
685 checksum0 = ~ip_csum_fold (sum0);
686 if (checksum0 != 0 && checksum0 != 0xffff)
688 next0 = SNAT_OUT2IN_NEXT_DROP;
692 old_addr0 = ip0->dst_address.as_u32;
693 new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
694 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
696 sum0 = ip0->checksum;
697 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
698 dst_address /* changed member */);
699 ip0->checksum = ip_csum_fold (sum0);
701 if (icmp0->checksum == 0)
702 icmp0->checksum = 0xffff;
704 if (!icmp_is_error_message (icmp0))
707 if (PREDICT_FALSE(new_id0 != echo0->identifier))
709 old_id0 = echo0->identifier;
711 echo0->identifier = new_id0;
713 sum0 = icmp0->checksum;
714 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
715 identifier /* changed member */);
716 icmp0->checksum = ip_csum_fold (sum0);
721 inner_ip0 = (ip4_header_t *)(echo0+1);
722 l4_header = ip4_next_header (inner_ip0);
724 if (!ip4_header_checksum_is_valid (inner_ip0))
726 next0 = SNAT_OUT2IN_NEXT_DROP;
730 old_addr0 = inner_ip0->src_address.as_u32;
731 inner_ip0->src_address = sm0.addr;
732 new_addr0 = inner_ip0->src_address.as_u32;
734 sum0 = icmp0->checksum;
735 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
736 src_address /* changed member */);
737 icmp0->checksum = ip_csum_fold (sum0);
741 case SNAT_PROTOCOL_ICMP:
742 inner_icmp0 = (icmp46_header_t*)l4_header;
743 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
745 old_id0 = inner_echo0->identifier;
747 inner_echo0->identifier = new_id0;
749 sum0 = icmp0->checksum;
750 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
752 icmp0->checksum = ip_csum_fold (sum0);
754 case SNAT_PROTOCOL_UDP:
755 case SNAT_PROTOCOL_TCP:
756 old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
758 ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
760 sum0 = icmp0->checksum;
761 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
763 icmp0->checksum = ip_csum_fold (sum0);
775 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
778 icmp46_header_t * icmp0,
781 vlib_node_runtime_t * node,
784 snat_session_t ** p_s0)
786 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
787 next0, thread_index, p_s0, 0);
788 snat_session_t * s0 = *p_s0;
789 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
792 nat44_session_update_counters (s0, now,
793 vlib_buffer_length_in_chain (sm->vlib_main, b0));
794 /* Per-user LRU list maintenance */
795 nat44_session_update_lru (sm, s0, thread_index);
800 static snat_session_t *
801 snat_out2in_unknown_proto (snat_main_t *sm,
808 vlib_node_runtime_t * node)
810 clib_bihash_kv_8_8_t kv, value;
811 clib_bihash_kv_16_8_t s_kv, s_value;
812 snat_static_mapping_t *m;
813 snat_session_key_t m_key;
814 u32 old_addr, new_addr;
816 nat_ed_ses_key_t key;
818 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
821 old_addr = ip->dst_address.as_u32;
823 key.l_addr = ip->dst_address;
824 key.r_addr = ip->src_address;
825 key.fib_index = rx_fib_index;
826 key.proto = ip->protocol;
829 s_kv.key[0] = key.as_u64[0];
830 s_kv.key[1] = key.as_u64[1];
832 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
834 s = pool_elt_at_index (tsm->sessions, s_value.value);
835 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
839 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
841 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
842 nat_log_notice ("maximum sessions exceeded");
846 m_key.addr = ip->dst_address;
849 m_key.fib_index = rx_fib_index;
850 kv.key = m_key.as_u64;
851 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
853 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
857 m = pool_elt_at_index (sm->static_mappings, value.value);
859 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
861 u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
865 nat_log_warn ("create NAT user failed");
869 /* Create a new session */
870 s = nat_session_alloc_or_recycle (sm, u, thread_index);
873 nat_log_warn ("create NAT session failed");
877 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
878 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
879 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
880 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
881 s->outside_address_index = ~0;
882 s->out2in.addr.as_u32 = old_addr;
883 s->out2in.fib_index = rx_fib_index;
884 s->in2out.addr.as_u32 = new_addr;
885 s->in2out.fib_index = m->fib_index;
886 s->in2out.port = s->out2in.port = ip->protocol;
887 user_session_increment (sm, u, 1 /* static */);
889 /* Add to lookup tables */
890 s_kv.value = s - tsm->sessions;
891 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
892 nat_log_notice ("out2in key add failed");
894 key.l_addr = ip->dst_address;
895 key.fib_index = m->fib_index;
896 s_kv.key[0] = key.as_u64[0];
897 s_kv.key[1] = key.as_u64[1];
898 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
899 nat_log_notice ("in2out key add failed");
902 /* Update IP checksum */
904 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
905 ip->checksum = ip_csum_fold (sum);
907 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
910 nat44_session_update_counters (s, now,
911 vlib_buffer_length_in_chain (vm, b));
912 /* Per-user LRU list maintenance */
913 nat44_session_update_lru (sm, s, thread_index);
918 static snat_session_t *
919 snat_out2in_lb (snat_main_t *sm,
926 vlib_node_runtime_t * node)
928 nat_ed_ses_key_t key;
929 clib_bihash_kv_16_8_t s_kv, s_value;
930 udp_header_t *udp = ip4_next_header (ip);
931 tcp_header_t *tcp = (tcp_header_t *) udp;
932 snat_session_t *s = 0;
933 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
934 snat_session_key_t e_key, l_key;
935 u32 old_addr, new_addr;
936 u32 proto = ip_proto_to_snat_proto (ip->protocol);
937 u16 new_port, old_port;
941 snat_session_key_t eh_key;
942 twice_nat_type_t twice_nat;
945 old_addr = ip->dst_address.as_u32;
947 key.l_addr = ip->dst_address;
948 key.r_addr = ip->src_address;
949 key.fib_index = rx_fib_index;
950 key.proto = ip->protocol;
951 key.r_port = udp->src_port;
952 key.l_port = udp->dst_port;
953 s_kv.key[0] = key.as_u64[0];
954 s_kv.key[1] = key.as_u64[1];
956 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
958 s = pool_elt_at_index (tsm->sessions, s_value.value);
962 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
964 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
965 nat_log_notice ("maximum sessions exceeded");
969 e_key.addr = ip->dst_address;
970 e_key.port = udp->dst_port;
971 e_key.protocol = proto;
972 e_key.fib_index = rx_fib_index;
973 if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat, &lb))
976 u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
980 nat_log_warn ("create NAT user failed");
984 s = nat_session_alloc_or_recycle (sm, u, thread_index);
987 nat_log_warn ("create NAT session failed");
991 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
992 s->ext_host_port = udp->src_port;
993 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
995 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
996 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
997 s->outside_address_index = ~0;
1000 user_session_increment (sm, u, 1 /* static */);
1002 /* Add to lookup tables */
1003 s_kv.value = s - tsm->sessions;
1004 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1005 nat_log_notice ("out2in-ed key add failed");
1007 if (twice_nat == TWICE_NAT ||
1008 (twice_nat == TWICE_NAT_SELF &&
1009 ip->src_address.as_u32 == l_key.addr.as_u32))
1011 eh_key.protocol = proto;
1012 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1013 thread_index, &eh_key,
1015 sm->port_per_thread,
1016 sm->per_thread_data[thread_index].snat_thread_index))
1018 b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1021 key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1022 key.r_port = s->ext_host_nat_port = eh_key.port;
1023 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1025 key.l_addr = l_key.addr;
1026 key.fib_index = l_key.fib_index;
1027 key.l_port = l_key.port;
1028 s_kv.key[0] = key.as_u64[0];
1029 s_kv.key[1] = key.as_u64[1];
1030 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1031 nat_log_notice ("in2out-ed key add failed");
1034 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1036 /* Update IP checksum */
1038 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1039 if (is_twice_nat_session (s))
1040 sum = ip_csum_update (sum, ip->src_address.as_u32,
1041 s->ext_host_nat_addr.as_u32, ip4_header_t,
1043 ip->checksum = ip_csum_fold (sum);
1045 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1047 if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1049 old_port = tcp->dst_port;
1050 tcp->dst_port = s->in2out.port;
1051 new_port = tcp->dst_port;
1053 sum = tcp->checksum;
1054 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1055 sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1056 if (is_twice_nat_session (s))
1058 sum = ip_csum_update (sum, ip->src_address.as_u32,
1059 s->ext_host_nat_addr.as_u32, ip4_header_t,
1061 sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1062 ip4_header_t, length);
1063 tcp->src_port = s->ext_host_nat_port;
1064 ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1066 tcp->checksum = ip_csum_fold(sum);
1067 if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
1072 udp->dst_port = s->in2out.port;
1073 if (is_twice_nat_session (s))
1075 udp->src_port = s->ext_host_nat_port;
1076 ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1082 nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
1083 /* Per-user LRU list maintenance */
1084 nat44_session_update_lru (sm, s, thread_index);
1090 snat_out2in_node_fn (vlib_main_t * vm,
1091 vlib_node_runtime_t * node,
1092 vlib_frame_t * frame)
1094 u32 n_left_from, * from, * to_next;
1095 snat_out2in_next_t next_index;
1096 u32 pkts_processed = 0;
1097 snat_main_t * sm = &snat_main;
1098 f64 now = vlib_time_now (vm);
1099 u32 thread_index = vlib_get_thread_index ();
1101 from = vlib_frame_vector_args (frame);
1102 n_left_from = frame->n_vectors;
1103 next_index = node->cached_next_index;
1105 while (n_left_from > 0)
1109 vlib_get_next_frame (vm, node, next_index,
1110 to_next, n_left_to_next);
1112 while (n_left_from >= 4 && n_left_to_next >= 2)
1115 vlib_buffer_t * b0, * b1;
1116 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1117 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1118 u32 sw_if_index0, sw_if_index1;
1119 ip4_header_t * ip0, *ip1;
1120 ip_csum_t sum0, sum1;
1121 u32 new_addr0, old_addr0;
1122 u16 new_port0, old_port0;
1123 u32 new_addr1, old_addr1;
1124 u16 new_port1, old_port1;
1125 udp_header_t * udp0, * udp1;
1126 tcp_header_t * tcp0, * tcp1;
1127 icmp46_header_t * icmp0, * icmp1;
1128 snat_session_key_t key0, key1, sm0, sm1;
1129 u32 rx_fib_index0, rx_fib_index1;
1131 snat_session_t * s0 = 0, * s1 = 0;
1132 clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1134 /* Prefetch next iteration. */
1136 vlib_buffer_t * p2, * p3;
1138 p2 = vlib_get_buffer (vm, from[2]);
1139 p3 = vlib_get_buffer (vm, from[3]);
1141 vlib_prefetch_buffer_header (p2, LOAD);
1142 vlib_prefetch_buffer_header (p3, LOAD);
1144 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1145 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1148 /* speculatively enqueue b0 and b1 to the current next frame */
1149 to_next[0] = bi0 = from[0];
1150 to_next[1] = bi1 = from[1];
1154 n_left_to_next -= 2;
1156 b0 = vlib_get_buffer (vm, bi0);
1157 b1 = vlib_get_buffer (vm, bi1);
1159 vnet_buffer (b0)->snat.flags = 0;
1160 vnet_buffer (b1)->snat.flags = 0;
1162 ip0 = vlib_buffer_get_current (b0);
1163 udp0 = ip4_next_header (ip0);
1164 tcp0 = (tcp_header_t *) udp0;
1165 icmp0 = (icmp46_header_t *) udp0;
1167 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1168 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1171 if (PREDICT_FALSE(ip0->ttl == 1))
1173 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1174 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1175 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1177 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1181 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1183 if (PREDICT_FALSE (proto0 == ~0))
1185 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1186 thread_index, now, vm, node);
1187 if (!sm->forwarding_enabled)
1189 next0 = SNAT_OUT2IN_NEXT_DROP;
1193 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1195 next0 = icmp_out2in_slow_path
1196 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1197 next0, now, thread_index, &s0);
1201 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1203 next0 = SNAT_OUT2IN_NEXT_REASS;
1207 key0.addr = ip0->dst_address;
1208 key0.port = udp0->dst_port;
1209 key0.protocol = proto0;
1210 key0.fib_index = rx_fib_index0;
1212 kv0.key = key0.as_u64;
1214 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1217 /* Try to match static mapping by external address and port,
1218 destination address and port in packet */
1219 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1222 * Send DHCP packets to the ipv4 stack, or we won't
1223 * be able to use dhcp client on the outside interface
1225 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1226 && (udp0->dst_port ==
1227 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1230 (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1234 if (!sm->forwarding_enabled)
1236 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1237 next0 = SNAT_OUT2IN_NEXT_DROP;
1242 if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1244 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1247 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1252 /* Create session initiated by host from external network */
1253 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1257 next0 = SNAT_OUT2IN_NEXT_DROP;
1263 if (PREDICT_FALSE (value0.value == ~0ULL))
1265 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1268 next0 = SNAT_OUT2IN_NEXT_DROP;
1273 s0 = pool_elt_at_index (
1274 sm->per_thread_data[thread_index].sessions,
1279 old_addr0 = ip0->dst_address.as_u32;
1280 ip0->dst_address = s0->in2out.addr;
1281 new_addr0 = ip0->dst_address.as_u32;
1282 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1284 sum0 = ip0->checksum;
1285 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1287 dst_address /* changed member */);
1288 ip0->checksum = ip_csum_fold (sum0);
1290 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1292 old_port0 = tcp0->dst_port;
1293 tcp0->dst_port = s0->in2out.port;
1294 new_port0 = tcp0->dst_port;
1296 sum0 = tcp0->checksum;
1297 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1299 dst_address /* changed member */);
1301 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1302 ip4_header_t /* cheat */,
1303 length /* changed member */);
1304 tcp0->checksum = ip_csum_fold(sum0);
1308 old_port0 = udp0->dst_port;
1309 udp0->dst_port = s0->in2out.port;
1314 nat44_session_update_counters (s0, now,
1315 vlib_buffer_length_in_chain (vm, b0));
1316 /* Per-user LRU list maintenance */
1317 nat44_session_update_lru (sm, s0, thread_index);
1320 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1321 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1323 snat_out2in_trace_t *t =
1324 vlib_add_trace (vm, node, b0, sizeof (*t));
1325 t->sw_if_index = sw_if_index0;
1326 t->next_index = next0;
1327 t->session_index = ~0;
1329 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1332 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1335 ip1 = vlib_buffer_get_current (b1);
1336 udp1 = ip4_next_header (ip1);
1337 tcp1 = (tcp_header_t *) udp1;
1338 icmp1 = (icmp46_header_t *) udp1;
1340 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1341 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1344 if (PREDICT_FALSE(ip1->ttl == 1))
1346 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1347 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1348 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1350 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1354 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1356 if (PREDICT_FALSE (proto1 == ~0))
1358 s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1359 thread_index, now, vm, node);
1360 if (!sm->forwarding_enabled)
1362 next1 = SNAT_OUT2IN_NEXT_DROP;
1366 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1368 next1 = icmp_out2in_slow_path
1369 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1370 next1, now, thread_index, &s1);
1374 if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1376 next1 = SNAT_OUT2IN_NEXT_REASS;
1380 key1.addr = ip1->dst_address;
1381 key1.port = udp1->dst_port;
1382 key1.protocol = proto1;
1383 key1.fib_index = rx_fib_index1;
1385 kv1.key = key1.as_u64;
1387 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1390 /* Try to match static mapping by external address and port,
1391 destination address and port in packet */
1392 if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
1395 * Send DHCP packets to the ipv4 stack, or we won't
1396 * be able to use dhcp client on the outside interface
1398 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1399 && (udp1->dst_port ==
1400 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1403 (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
1407 if (!sm->forwarding_enabled)
1409 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1410 next1 = SNAT_OUT2IN_NEXT_DROP;
1415 if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1417 next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1420 create_bypass_for_fwd(sm, ip1, rx_fib_index1, thread_index);
1425 /* Create session initiated by host from external network */
1426 s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1430 next1 = SNAT_OUT2IN_NEXT_DROP;
1436 if (PREDICT_FALSE (value1.value == ~0ULL))
1438 s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1441 next1 = SNAT_OUT2IN_NEXT_DROP;
1446 s1 = pool_elt_at_index (
1447 sm->per_thread_data[thread_index].sessions,
1452 old_addr1 = ip1->dst_address.as_u32;
1453 ip1->dst_address = s1->in2out.addr;
1454 new_addr1 = ip1->dst_address.as_u32;
1455 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1457 sum1 = ip1->checksum;
1458 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1460 dst_address /* changed member */);
1461 ip1->checksum = ip_csum_fold (sum1);
1463 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1465 old_port1 = tcp1->dst_port;
1466 tcp1->dst_port = s1->in2out.port;
1467 new_port1 = tcp1->dst_port;
1469 sum1 = tcp1->checksum;
1470 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1472 dst_address /* changed member */);
1474 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1475 ip4_header_t /* cheat */,
1476 length /* changed member */);
1477 tcp1->checksum = ip_csum_fold(sum1);
1481 old_port1 = udp1->dst_port;
1482 udp1->dst_port = s1->in2out.port;
1487 nat44_session_update_counters (s1, now,
1488 vlib_buffer_length_in_chain (vm, b1));
1489 /* Per-user LRU list maintenance */
1490 nat44_session_update_lru (sm, s1, thread_index);
1493 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1494 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1496 snat_out2in_trace_t *t =
1497 vlib_add_trace (vm, node, b1, sizeof (*t));
1498 t->sw_if_index = sw_if_index1;
1499 t->next_index = next1;
1500 t->session_index = ~0;
1502 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1505 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1507 /* verify speculative enqueues, maybe switch current next frame */
1508 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1509 to_next, n_left_to_next,
1510 bi0, bi1, next0, next1);
1513 while (n_left_from > 0 && n_left_to_next > 0)
1517 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1521 u32 new_addr0, old_addr0;
1522 u16 new_port0, old_port0;
1523 udp_header_t * udp0;
1524 tcp_header_t * tcp0;
1525 icmp46_header_t * icmp0;
1526 snat_session_key_t key0, sm0;
1529 snat_session_t * s0 = 0;
1530 clib_bihash_kv_8_8_t kv0, value0;
1532 /* speculatively enqueue b0 to the current next frame */
1538 n_left_to_next -= 1;
1540 b0 = vlib_get_buffer (vm, bi0);
1542 vnet_buffer (b0)->snat.flags = 0;
1544 ip0 = vlib_buffer_get_current (b0);
1545 udp0 = ip4_next_header (ip0);
1546 tcp0 = (tcp_header_t *) udp0;
1547 icmp0 = (icmp46_header_t *) udp0;
1549 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1550 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1553 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1555 if (PREDICT_FALSE (proto0 == ~0))
1557 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1558 thread_index, now, vm, node);
1559 if (!sm->forwarding_enabled)
1561 next0 = SNAT_OUT2IN_NEXT_DROP;
1565 if (PREDICT_FALSE(ip0->ttl == 1))
1567 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1568 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1569 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1571 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1575 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1577 next0 = icmp_out2in_slow_path
1578 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1579 next0, now, thread_index, &s0);
1583 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1585 next0 = SNAT_OUT2IN_NEXT_REASS;
1589 key0.addr = ip0->dst_address;
1590 key0.port = udp0->dst_port;
1591 key0.protocol = proto0;
1592 key0.fib_index = rx_fib_index0;
1594 kv0.key = key0.as_u64;
1596 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1599 /* Try to match static mapping by external address and port,
1600 destination address and port in packet */
1601 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1604 * Send DHCP packets to the ipv4 stack, or we won't
1605 * be able to use dhcp client on the outside interface
1607 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1608 && (udp0->dst_port ==
1609 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1612 (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1616 if (!sm->forwarding_enabled)
1618 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1619 next0 = SNAT_OUT2IN_NEXT_DROP;
1624 if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1626 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1629 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1634 /* Create session initiated by host from external network */
1635 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1639 next0 = SNAT_OUT2IN_NEXT_DROP;
1645 if (PREDICT_FALSE (value0.value == ~0ULL))
1647 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1650 next0 = SNAT_OUT2IN_NEXT_DROP;
1655 s0 = pool_elt_at_index (
1656 sm->per_thread_data[thread_index].sessions,
1661 old_addr0 = ip0->dst_address.as_u32;
1662 ip0->dst_address = s0->in2out.addr;
1663 new_addr0 = ip0->dst_address.as_u32;
1664 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1666 sum0 = ip0->checksum;
1667 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1669 dst_address /* changed member */);
1670 ip0->checksum = ip_csum_fold (sum0);
1672 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1674 old_port0 = tcp0->dst_port;
1675 tcp0->dst_port = s0->in2out.port;
1676 new_port0 = tcp0->dst_port;
1678 sum0 = tcp0->checksum;
1679 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1681 dst_address /* changed member */);
1683 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1684 ip4_header_t /* cheat */,
1685 length /* changed member */);
1686 tcp0->checksum = ip_csum_fold(sum0);
1690 old_port0 = udp0->dst_port;
1691 udp0->dst_port = s0->in2out.port;
1696 nat44_session_update_counters (s0, now,
1697 vlib_buffer_length_in_chain (vm, b0));
1698 /* Per-user LRU list maintenance */
1699 nat44_session_update_lru (sm, s0, thread_index);
1702 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1703 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1705 snat_out2in_trace_t *t =
1706 vlib_add_trace (vm, node, b0, sizeof (*t));
1707 t->sw_if_index = sw_if_index0;
1708 t->next_index = next0;
1709 t->session_index = ~0;
1711 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1714 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1716 /* verify speculative enqueue, maybe switch current next frame */
1717 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1718 to_next, n_left_to_next,
1722 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1725 vlib_node_increment_counter (vm, snat_out2in_node.index,
1726 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1728 return frame->n_vectors;
1731 VLIB_REGISTER_NODE (snat_out2in_node) = {
1732 .function = snat_out2in_node_fn,
1733 .name = "nat44-out2in",
1734 .vector_size = sizeof (u32),
1735 .format_trace = format_snat_out2in_trace,
1736 .type = VLIB_NODE_TYPE_INTERNAL,
1738 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1739 .error_strings = snat_out2in_error_strings,
1741 .runtime_data_bytes = sizeof (snat_runtime_t),
1743 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1745 /* edit / add dispositions here */
1747 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1748 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1749 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1750 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1751 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1754 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1757 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1758 vlib_node_runtime_t * node,
1759 vlib_frame_t * frame)
1761 u32 n_left_from, *from, *to_next;
1762 snat_out2in_next_t next_index;
1763 u32 pkts_processed = 0;
1764 snat_main_t *sm = &snat_main;
1765 f64 now = vlib_time_now (vm);
1766 u32 thread_index = vlib_get_thread_index ();
1767 snat_main_per_thread_data_t *per_thread_data =
1768 &sm->per_thread_data[thread_index];
1769 u32 *fragments_to_drop = 0;
1770 u32 *fragments_to_loopback = 0;
1772 from = vlib_frame_vector_args (frame);
1773 n_left_from = frame->n_vectors;
1774 next_index = node->cached_next_index;
1776 while (n_left_from > 0)
1780 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1782 while (n_left_from > 0 && n_left_to_next > 0)
1784 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1789 nat_reass_ip4_t *reass0;
1790 udp_header_t * udp0;
1791 tcp_header_t * tcp0;
1792 snat_session_key_t key0, sm0;
1793 clib_bihash_kv_8_8_t kv0, value0;
1794 snat_session_t * s0 = 0;
1795 u16 old_port0, new_port0;
1798 /* speculatively enqueue b0 to the current next frame */
1804 n_left_to_next -= 1;
1806 b0 = vlib_get_buffer (vm, bi0);
1807 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1809 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1810 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1813 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1815 next0 = SNAT_OUT2IN_NEXT_DROP;
1816 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1820 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1821 udp0 = ip4_next_header (ip0);
1822 tcp0 = (tcp_header_t *) udp0;
1823 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1825 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1830 &fragments_to_drop);
1832 if (PREDICT_FALSE (!reass0))
1834 next0 = SNAT_OUT2IN_NEXT_DROP;
1835 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1836 nat_log_notice ("maximum reassemblies exceeded");
1840 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1842 key0.addr = ip0->dst_address;
1843 key0.port = udp0->dst_port;
1844 key0.protocol = proto0;
1845 key0.fib_index = rx_fib_index0;
1846 kv0.key = key0.as_u64;
1848 if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1850 /* Try to match static mapping by external address and port,
1851 destination address and port in packet */
1852 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1855 * Send DHCP packets to the ipv4 stack, or we won't
1856 * be able to use dhcp client on the outside interface
1858 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1860 == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1863 (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1868 if (!sm->forwarding_enabled)
1870 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1871 next0 = SNAT_OUT2IN_NEXT_DROP;
1876 if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1878 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1881 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1886 /* Create session initiated by host from external network */
1887 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1891 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1892 next0 = SNAT_OUT2IN_NEXT_DROP;
1895 reass0->sess_index = s0 - per_thread_data->sessions;
1896 reass0->thread_index = thread_index;
1900 s0 = pool_elt_at_index (per_thread_data->sessions,
1902 reass0->sess_index = value0.value;
1904 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1908 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1910 if (nat_ip4_reass_add_fragment (reass0, bi0))
1912 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1913 nat_log_notice ("maximum fragments per reassembly exceeded");
1914 next0 = SNAT_OUT2IN_NEXT_DROP;
1920 s0 = pool_elt_at_index (per_thread_data->sessions,
1921 reass0->sess_index);
1924 old_addr0 = ip0->dst_address.as_u32;
1925 ip0->dst_address = s0->in2out.addr;
1926 new_addr0 = ip0->dst_address.as_u32;
1927 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1929 sum0 = ip0->checksum;
1930 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1932 dst_address /* changed member */);
1933 ip0->checksum = ip_csum_fold (sum0);
1935 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1937 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1939 old_port0 = tcp0->dst_port;
1940 tcp0->dst_port = s0->in2out.port;
1941 new_port0 = tcp0->dst_port;
1943 sum0 = tcp0->checksum;
1944 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1946 dst_address /* changed member */);
1948 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1949 ip4_header_t /* cheat */,
1950 length /* changed member */);
1951 tcp0->checksum = ip_csum_fold(sum0);
1955 old_port0 = udp0->dst_port;
1956 udp0->dst_port = s0->in2out.port;
1962 nat44_session_update_counters (s0, now,
1963 vlib_buffer_length_in_chain (vm, b0));
1964 /* Per-user LRU list maintenance */
1965 nat44_session_update_lru (sm, s0, thread_index);
1968 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1969 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1971 nat44_out2in_reass_trace_t *t =
1972 vlib_add_trace (vm, node, b0, sizeof (*t));
1973 t->cached = cached0;
1974 t->sw_if_index = sw_if_index0;
1975 t->next_index = next0;
1985 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1987 /* verify speculative enqueue, maybe switch current next frame */
1988 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1989 to_next, n_left_to_next,
1993 if (n_left_from == 0 && vec_len (fragments_to_loopback))
1995 from = vlib_frame_vector_args (frame);
1996 u32 len = vec_len (fragments_to_loopback);
1997 if (len <= VLIB_FRAME_SIZE)
1999 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2001 vec_reset_length (fragments_to_loopback);
2006 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2007 sizeof (u32) * VLIB_FRAME_SIZE);
2008 n_left_from = VLIB_FRAME_SIZE;
2009 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2014 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2017 vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
2018 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2021 nat_send_all_to_node (vm, fragments_to_drop, node,
2022 &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
2023 SNAT_OUT2IN_NEXT_DROP);
2025 vec_free (fragments_to_drop);
2026 vec_free (fragments_to_loopback);
2027 return frame->n_vectors;
2030 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
2031 .function = nat44_out2in_reass_node_fn,
2032 .name = "nat44-out2in-reass",
2033 .vector_size = sizeof (u32),
2034 .format_trace = format_nat44_out2in_reass_trace,
2035 .type = VLIB_NODE_TYPE_INTERNAL,
2037 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2038 .error_strings = snat_out2in_error_strings,
2040 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2042 /* edit / add dispositions here */
2044 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2045 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2046 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2047 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2048 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2051 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2052 nat44_out2in_reass_node_fn);
2054 /**************************/
2055 /*** deterministic mode ***/
2056 /**************************/
2058 snat_det_out2in_node_fn (vlib_main_t * vm,
2059 vlib_node_runtime_t * node,
2060 vlib_frame_t * frame)
2062 u32 n_left_from, * from, * to_next;
2063 snat_out2in_next_t next_index;
2064 u32 pkts_processed = 0;
2065 snat_main_t * sm = &snat_main;
2066 u32 thread_index = vlib_get_thread_index ();
2068 from = vlib_frame_vector_args (frame);
2069 n_left_from = frame->n_vectors;
2070 next_index = node->cached_next_index;
2072 while (n_left_from > 0)
2076 vlib_get_next_frame (vm, node, next_index,
2077 to_next, n_left_to_next);
2079 while (n_left_from >= 4 && n_left_to_next >= 2)
2082 vlib_buffer_t * b0, * b1;
2083 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2084 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2085 u32 sw_if_index0, sw_if_index1;
2086 ip4_header_t * ip0, * ip1;
2087 ip_csum_t sum0, sum1;
2088 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2089 u16 new_port0, old_port0, old_port1, new_port1;
2090 udp_header_t * udp0, * udp1;
2091 tcp_header_t * tcp0, * tcp1;
2093 snat_det_out_key_t key0, key1;
2094 snat_det_map_t * dm0, * dm1;
2095 snat_det_session_t * ses0 = 0, * ses1 = 0;
2096 u32 rx_fib_index0, rx_fib_index1;
2097 icmp46_header_t * icmp0, * icmp1;
2099 /* Prefetch next iteration. */
2101 vlib_buffer_t * p2, * p3;
2103 p2 = vlib_get_buffer (vm, from[2]);
2104 p3 = vlib_get_buffer (vm, from[3]);
2106 vlib_prefetch_buffer_header (p2, LOAD);
2107 vlib_prefetch_buffer_header (p3, LOAD);
2109 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2110 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2113 /* speculatively enqueue b0 and b1 to the current next frame */
2114 to_next[0] = bi0 = from[0];
2115 to_next[1] = bi1 = from[1];
2119 n_left_to_next -= 2;
2121 b0 = vlib_get_buffer (vm, bi0);
2122 b1 = vlib_get_buffer (vm, bi1);
2124 ip0 = vlib_buffer_get_current (b0);
2125 udp0 = ip4_next_header (ip0);
2126 tcp0 = (tcp_header_t *) udp0;
2128 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2130 if (PREDICT_FALSE(ip0->ttl == 1))
2132 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2133 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2134 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2136 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2140 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2142 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2144 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2145 icmp0 = (icmp46_header_t *) udp0;
2147 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2148 rx_fib_index0, node, next0, thread_index,
2153 key0.ext_host_addr = ip0->src_address;
2154 key0.ext_host_port = tcp0->src;
2155 key0.out_port = tcp0->dst;
2157 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2158 if (PREDICT_FALSE(!dm0))
2160 nat_log_info ("unknown dst address: %U",
2161 format_ip4_address, &ip0->dst_address);
2162 next0 = SNAT_OUT2IN_NEXT_DROP;
2163 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2167 snat_det_reverse(dm0, &ip0->dst_address,
2168 clib_net_to_host_u16(tcp0->dst), &new_addr0);
2170 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2171 if (PREDICT_FALSE(!ses0))
2173 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2174 format_ip4_address, &ip0->src_address,
2175 clib_net_to_host_u16 (tcp0->src),
2176 format_ip4_address, &ip0->dst_address,
2177 clib_net_to_host_u16 (tcp0->dst),
2178 format_ip4_address, &new_addr0);
2179 next0 = SNAT_OUT2IN_NEXT_DROP;
2180 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2183 new_port0 = ses0->in_port;
2185 old_addr0 = ip0->dst_address;
2186 ip0->dst_address = new_addr0;
2187 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2189 sum0 = ip0->checksum;
2190 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2192 dst_address /* changed member */);
2193 ip0->checksum = ip_csum_fold (sum0);
2195 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2197 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2198 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2199 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2200 snat_det_ses_close(dm0, ses0);
2202 old_port0 = tcp0->dst;
2203 tcp0->dst = new_port0;
2205 sum0 = tcp0->checksum;
2206 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2208 dst_address /* changed member */);
2210 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2211 ip4_header_t /* cheat */,
2212 length /* changed member */);
2213 tcp0->checksum = ip_csum_fold(sum0);
2217 old_port0 = udp0->dst_port;
2218 udp0->dst_port = new_port0;
2224 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2225 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2227 snat_out2in_trace_t *t =
2228 vlib_add_trace (vm, node, b0, sizeof (*t));
2229 t->sw_if_index = sw_if_index0;
2230 t->next_index = next0;
2231 t->session_index = ~0;
2233 t->session_index = ses0 - dm0->sessions;
2236 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2238 b1 = vlib_get_buffer (vm, bi1);
2240 ip1 = vlib_buffer_get_current (b1);
2241 udp1 = ip4_next_header (ip1);
2242 tcp1 = (tcp_header_t *) udp1;
2244 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2246 if (PREDICT_FALSE(ip1->ttl == 1))
2248 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2249 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2250 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2252 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2256 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2258 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2260 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2261 icmp1 = (icmp46_header_t *) udp1;
2263 next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2264 rx_fib_index1, node, next1, thread_index,
2269 key1.ext_host_addr = ip1->src_address;
2270 key1.ext_host_port = tcp1->src;
2271 key1.out_port = tcp1->dst;
2273 dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2274 if (PREDICT_FALSE(!dm1))
2276 nat_log_info ("unknown dst address: %U",
2277 format_ip4_address, &ip1->dst_address);
2278 next1 = SNAT_OUT2IN_NEXT_DROP;
2279 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2283 snat_det_reverse(dm1, &ip1->dst_address,
2284 clib_net_to_host_u16(tcp1->dst), &new_addr1);
2286 ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2287 if (PREDICT_FALSE(!ses1))
2289 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2290 format_ip4_address, &ip1->src_address,
2291 clib_net_to_host_u16 (tcp1->src),
2292 format_ip4_address, &ip1->dst_address,
2293 clib_net_to_host_u16 (tcp1->dst),
2294 format_ip4_address, &new_addr1);
2295 next1 = SNAT_OUT2IN_NEXT_DROP;
2296 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2299 new_port1 = ses1->in_port;
2301 old_addr1 = ip1->dst_address;
2302 ip1->dst_address = new_addr1;
2303 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2305 sum1 = ip1->checksum;
2306 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2308 dst_address /* changed member */);
2309 ip1->checksum = ip_csum_fold (sum1);
2311 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2313 if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2314 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2315 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2316 snat_det_ses_close(dm1, ses1);
2318 old_port1 = tcp1->dst;
2319 tcp1->dst = new_port1;
2321 sum1 = tcp1->checksum;
2322 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2324 dst_address /* changed member */);
2326 sum1 = ip_csum_update (sum1, old_port1, new_port1,
2327 ip4_header_t /* cheat */,
2328 length /* changed member */);
2329 tcp1->checksum = ip_csum_fold(sum1);
2333 old_port1 = udp1->dst_port;
2334 udp1->dst_port = new_port1;
2340 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2341 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2343 snat_out2in_trace_t *t =
2344 vlib_add_trace (vm, node, b1, sizeof (*t));
2345 t->sw_if_index = sw_if_index1;
2346 t->next_index = next1;
2347 t->session_index = ~0;
2349 t->session_index = ses1 - dm1->sessions;
2352 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2354 /* verify speculative enqueues, maybe switch current next frame */
2355 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2356 to_next, n_left_to_next,
2357 bi0, bi1, next0, next1);
2360 while (n_left_from > 0 && n_left_to_next > 0)
2364 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2368 ip4_address_t new_addr0, old_addr0;
2369 u16 new_port0, old_port0;
2370 udp_header_t * udp0;
2371 tcp_header_t * tcp0;
2373 snat_det_out_key_t key0;
2374 snat_det_map_t * dm0;
2375 snat_det_session_t * ses0 = 0;
2377 icmp46_header_t * icmp0;
2379 /* speculatively enqueue b0 to the current next frame */
2385 n_left_to_next -= 1;
2387 b0 = vlib_get_buffer (vm, bi0);
2389 ip0 = vlib_buffer_get_current (b0);
2390 udp0 = ip4_next_header (ip0);
2391 tcp0 = (tcp_header_t *) udp0;
2393 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2395 if (PREDICT_FALSE(ip0->ttl == 1))
2397 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2398 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2399 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2401 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2405 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2407 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2409 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2410 icmp0 = (icmp46_header_t *) udp0;
2412 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2413 rx_fib_index0, node, next0, thread_index,
2418 key0.ext_host_addr = ip0->src_address;
2419 key0.ext_host_port = tcp0->src;
2420 key0.out_port = tcp0->dst;
2422 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2423 if (PREDICT_FALSE(!dm0))
2425 nat_log_info ("unknown dst address: %U",
2426 format_ip4_address, &ip0->dst_address);
2427 next0 = SNAT_OUT2IN_NEXT_DROP;
2428 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2432 snat_det_reverse(dm0, &ip0->dst_address,
2433 clib_net_to_host_u16(tcp0->dst), &new_addr0);
2435 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2436 if (PREDICT_FALSE(!ses0))
2438 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2439 format_ip4_address, &ip0->src_address,
2440 clib_net_to_host_u16 (tcp0->src),
2441 format_ip4_address, &ip0->dst_address,
2442 clib_net_to_host_u16 (tcp0->dst),
2443 format_ip4_address, &new_addr0);
2444 next0 = SNAT_OUT2IN_NEXT_DROP;
2445 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2448 new_port0 = ses0->in_port;
2450 old_addr0 = ip0->dst_address;
2451 ip0->dst_address = new_addr0;
2452 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2454 sum0 = ip0->checksum;
2455 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2457 dst_address /* changed member */);
2458 ip0->checksum = ip_csum_fold (sum0);
2460 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2462 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2463 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2464 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2465 snat_det_ses_close(dm0, ses0);
2467 old_port0 = tcp0->dst;
2468 tcp0->dst = new_port0;
2470 sum0 = tcp0->checksum;
2471 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2473 dst_address /* changed member */);
2475 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2476 ip4_header_t /* cheat */,
2477 length /* changed member */);
2478 tcp0->checksum = ip_csum_fold(sum0);
2482 old_port0 = udp0->dst_port;
2483 udp0->dst_port = new_port0;
2489 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2490 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2492 snat_out2in_trace_t *t =
2493 vlib_add_trace (vm, node, b0, sizeof (*t));
2494 t->sw_if_index = sw_if_index0;
2495 t->next_index = next0;
2496 t->session_index = ~0;
2498 t->session_index = ses0 - dm0->sessions;
2501 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2503 /* verify speculative enqueue, maybe switch current next frame */
2504 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2505 to_next, n_left_to_next,
2509 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2512 vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2513 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2515 return frame->n_vectors;
2518 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2519 .function = snat_det_out2in_node_fn,
2520 .name = "nat44-det-out2in",
2521 .vector_size = sizeof (u32),
2522 .format_trace = format_snat_out2in_trace,
2523 .type = VLIB_NODE_TYPE_INTERNAL,
2525 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2526 .error_strings = snat_out2in_error_strings,
2528 .runtime_data_bytes = sizeof (snat_runtime_t),
2530 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2532 /* edit / add dispositions here */
2534 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2535 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2536 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2537 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2538 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2541 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2544 * Get address and port values to be used for ICMP packet translation
2545 * and create session if needed
2547 * @param[in,out] sm NAT main
2548 * @param[in,out] node NAT node runtime
2549 * @param[in] thread_index thread index
2550 * @param[in,out] b0 buffer containing packet to be translated
2551 * @param[out] p_proto protocol used for matching
2552 * @param[out] p_value address and port after NAT translation
2553 * @param[out] p_dont_translate if packet should not be translated
2554 * @param d optional parameter
2555 * @param e optional parameter
2557 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2558 u32 thread_index, vlib_buffer_t *b0,
2559 ip4_header_t *ip0, u8 *p_proto,
2560 snat_session_key_t *p_value,
2561 u8 *p_dont_translate, void *d, void *e)
2563 icmp46_header_t *icmp0;
2566 snat_det_out_key_t key0;
2567 u8 dont_translate = 0;
2569 icmp_echo_header_t *echo0, *inner_echo0 = 0;
2570 ip4_header_t *inner_ip0;
2571 void *l4_header = 0;
2572 icmp46_header_t *inner_icmp0;
2573 snat_det_map_t * dm0 = 0;
2574 ip4_address_t new_addr0 = {{0}};
2575 snat_det_session_t * ses0 = 0;
2576 ip4_address_t out_addr;
2578 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2579 echo0 = (icmp_echo_header_t *)(icmp0+1);
2580 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2582 if (!icmp_is_error_message (icmp0))
2584 protocol = SNAT_PROTOCOL_ICMP;
2585 key0.ext_host_addr = ip0->src_address;
2586 key0.ext_host_port = 0;
2587 key0.out_port = echo0->identifier;
2588 out_addr = ip0->dst_address;
2592 inner_ip0 = (ip4_header_t *)(echo0+1);
2593 l4_header = ip4_next_header (inner_ip0);
2594 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2595 key0.ext_host_addr = inner_ip0->dst_address;
2596 out_addr = inner_ip0->src_address;
2599 case SNAT_PROTOCOL_ICMP:
2600 inner_icmp0 = (icmp46_header_t*)l4_header;
2601 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2602 key0.ext_host_port = 0;
2603 key0.out_port = inner_echo0->identifier;
2605 case SNAT_PROTOCOL_UDP:
2606 case SNAT_PROTOCOL_TCP:
2607 key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2608 key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2611 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2612 next0 = SNAT_OUT2IN_NEXT_DROP;
2617 dm0 = snat_det_map_by_out(sm, &out_addr);
2618 if (PREDICT_FALSE(!dm0))
2620 /* Don't NAT packet aimed at the intfc address */
2621 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2622 ip0->dst_address.as_u32)))
2627 nat_log_info ("unknown dst address: %U",
2628 format_ip4_address, &ip0->dst_address);
2632 snat_det_reverse(dm0, &ip0->dst_address,
2633 clib_net_to_host_u16(key0.out_port), &new_addr0);
2635 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2636 if (PREDICT_FALSE(!ses0))
2638 /* Don't NAT packet aimed at the intfc address */
2639 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2640 ip0->dst_address.as_u32)))
2645 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2646 format_ip4_address, &key0.ext_host_addr,
2647 clib_net_to_host_u16 (key0.ext_host_port),
2648 format_ip4_address, &out_addr,
2649 clib_net_to_host_u16 (key0.out_port),
2650 format_ip4_address, &new_addr0);
2651 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2652 next0 = SNAT_OUT2IN_NEXT_DROP;
2656 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2657 !icmp_is_error_message (icmp0)))
2659 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2660 next0 = SNAT_OUT2IN_NEXT_DROP;
2667 *p_proto = protocol;
2670 p_value->addr = new_addr0;
2671 p_value->fib_index = sm->inside_fib_index;
2672 p_value->port = ses0->in_port;
2674 *p_dont_translate = dont_translate;
2676 *(snat_det_session_t**)d = ses0;
2678 *(snat_det_map_t**)e = dm0;
2682 /**********************/
2683 /*** worker handoff ***/
2684 /**********************/
2686 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2687 vlib_node_runtime_t * node,
2688 vlib_frame_t * frame)
2690 snat_main_t *sm = &snat_main;
2691 vlib_thread_main_t *tm = vlib_get_thread_main ();
2692 u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
2693 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2694 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2696 vlib_frame_queue_elt_t *hf = 0;
2697 vlib_frame_queue_t *fq;
2698 vlib_frame_t *f = 0;
2700 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2701 u32 next_worker_index = 0;
2702 u32 current_worker_index = ~0;
2703 u32 thread_index = vlib_get_thread_index ();
2704 vlib_frame_t *d = 0;
2706 ASSERT (vec_len (sm->workers));
2708 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2710 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2712 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2713 tm->n_vlib_mains - 1,
2714 (vlib_frame_queue_t *) (~0));
2717 from = vlib_frame_vector_args (frame);
2718 n_left_from = frame->n_vectors;
2720 while (n_left_from > 0)
2733 b0 = vlib_get_buffer (vm, bi0);
2735 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2736 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2738 ip0 = vlib_buffer_get_current (b0);
2740 next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2742 if (PREDICT_FALSE (next_worker_index != thread_index))
2746 if (next_worker_index != current_worker_index)
2748 fq = is_vlib_frame_queue_congested (
2749 sm->fq_out2in_index, next_worker_index, NAT_FQ_NELTS - 2,
2750 congested_handoff_queue_by_worker_index);
2754 /* if this is 1st frame */
2757 d = vlib_get_frame_to_node (vm, sm->error_node_index);
2758 to_next_drop = vlib_frame_vector_args (d);
2761 to_next_drop[0] = bi0;
2764 b0->error = node->errors[SNAT_OUT2IN_ERROR_FQ_CONGESTED];
2769 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2771 hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2773 handoff_queue_elt_by_worker_index);
2775 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2776 to_next_worker = &hf->buffer_index[hf->n_vectors];
2777 current_worker_index = next_worker_index;
2780 /* enqueue to correct worker thread */
2781 to_next_worker[0] = bi0;
2783 n_left_to_next_worker--;
2785 if (n_left_to_next_worker == 0)
2787 hf->n_vectors = VLIB_FRAME_SIZE;
2788 vlib_put_frame_queue_elt (hf);
2789 current_worker_index = ~0;
2790 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2797 /* if this is 1st frame */
2800 f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2801 to_next = vlib_frame_vector_args (f);
2810 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2811 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2813 snat_out2in_worker_handoff_trace_t *t =
2814 vlib_add_trace (vm, node, b0, sizeof (*t));
2815 t->next_worker_index = next_worker_index;
2816 t->do_handoff = do_handoff;
2821 vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2824 vlib_put_frame_to_node (vm, sm->error_node_index, d);
2827 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2829 /* Ship frames to the worker nodes */
2830 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2832 if (handoff_queue_elt_by_worker_index[i])
2834 hf = handoff_queue_elt_by_worker_index[i];
2836 * It works better to let the handoff node
2837 * rate-adapt, always ship the handoff queue element.
2839 if (1 || hf->n_vectors == hf->last_n_vectors)
2841 vlib_put_frame_queue_elt (hf);
2842 handoff_queue_elt_by_worker_index[i] = 0;
2845 hf->last_n_vectors = hf->n_vectors;
2847 congested_handoff_queue_by_worker_index[i] =
2848 (vlib_frame_queue_t *) (~0);
2851 current_worker_index = ~0;
2852 return frame->n_vectors;
2855 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2856 .function = snat_out2in_worker_handoff_fn,
2857 .name = "nat44-out2in-worker-handoff",
2858 .vector_size = sizeof (u32),
2859 .format_trace = format_snat_out2in_worker_handoff_trace,
2860 .type = VLIB_NODE_TYPE_INTERNAL,
2862 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2863 .error_strings = snat_out2in_error_strings,
2872 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2875 snat_out2in_fast_node_fn (vlib_main_t * vm,
2876 vlib_node_runtime_t * node,
2877 vlib_frame_t * frame)
2879 u32 n_left_from, * from, * to_next;
2880 snat_out2in_next_t next_index;
2881 u32 pkts_processed = 0;
2882 snat_main_t * sm = &snat_main;
2884 from = vlib_frame_vector_args (frame);
2885 n_left_from = frame->n_vectors;
2886 next_index = node->cached_next_index;
2888 while (n_left_from > 0)
2892 vlib_get_next_frame (vm, node, next_index,
2893 to_next, n_left_to_next);
2895 while (n_left_from > 0 && n_left_to_next > 0)
2899 u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2903 u32 new_addr0, old_addr0;
2904 u16 new_port0, old_port0;
2905 udp_header_t * udp0;
2906 tcp_header_t * tcp0;
2907 icmp46_header_t * icmp0;
2908 snat_session_key_t key0, sm0;
2912 /* speculatively enqueue b0 to the current next frame */
2918 n_left_to_next -= 1;
2920 b0 = vlib_get_buffer (vm, bi0);
2922 ip0 = vlib_buffer_get_current (b0);
2923 udp0 = ip4_next_header (ip0);
2924 tcp0 = (tcp_header_t *) udp0;
2925 icmp0 = (icmp46_header_t *) udp0;
2927 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2928 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2930 vnet_feature_next (sw_if_index0, &next0, b0);
2932 if (PREDICT_FALSE(ip0->ttl == 1))
2934 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2935 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2936 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2938 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2942 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2944 if (PREDICT_FALSE (proto0 == ~0))
2947 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2949 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2950 rx_fib_index0, node, next0, ~0, 0, 0);
2954 key0.addr = ip0->dst_address;
2955 key0.port = udp0->dst_port;
2956 key0.fib_index = rx_fib_index0;
2958 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2960 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2964 new_addr0 = sm0.addr.as_u32;
2965 new_port0 = sm0.port;
2966 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2967 old_addr0 = ip0->dst_address.as_u32;
2968 ip0->dst_address.as_u32 = new_addr0;
2970 sum0 = ip0->checksum;
2971 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2973 dst_address /* changed member */);
2974 ip0->checksum = ip_csum_fold (sum0);
2976 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2978 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2980 old_port0 = tcp0->dst_port;
2981 tcp0->dst_port = new_port0;
2983 sum0 = tcp0->checksum;
2984 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2986 dst_address /* changed member */);
2988 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2989 ip4_header_t /* cheat */,
2990 length /* changed member */);
2991 tcp0->checksum = ip_csum_fold(sum0);
2995 old_port0 = udp0->dst_port;
2996 udp0->dst_port = new_port0;
3002 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3004 sum0 = tcp0->checksum;
3005 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3007 dst_address /* changed member */);
3009 tcp0->checksum = ip_csum_fold(sum0);
3015 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3016 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3018 snat_out2in_trace_t *t =
3019 vlib_add_trace (vm, node, b0, sizeof (*t));
3020 t->sw_if_index = sw_if_index0;
3021 t->next_index = next0;
3024 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3026 /* verify speculative enqueue, maybe switch current next frame */
3027 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3028 to_next, n_left_to_next,
3032 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3035 vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3036 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3038 return frame->n_vectors;
3041 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3042 .function = snat_out2in_fast_node_fn,
3043 .name = "nat44-out2in-fast",
3044 .vector_size = sizeof (u32),
3045 .format_trace = format_snat_out2in_fast_trace,
3046 .type = VLIB_NODE_TYPE_INTERNAL,
3048 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3049 .error_strings = snat_out2in_error_strings,
3051 .runtime_data_bytes = sizeof (snat_runtime_t),
3053 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3055 /* edit / add dispositions here */
3057 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3058 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3059 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3060 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3061 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
3064 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);