2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
26 #include <nat/nat_ipfix_logging.h>
27 #include <nat/nat_det.h>
28 #include <nat/nat_reass.h>
30 #include <vppinfra/hash.h>
31 #include <vppinfra/error.h>
32 #include <vppinfra/elog.h>
38 } snat_out2in_trace_t;
41 u32 next_worker_index;
43 } snat_out2in_worker_handoff_trace_t;
45 /* packet trace format function */
46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
48 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
49 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
50 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
52 s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
53 t->sw_if_index, t->next_index, t->session_index);
57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
59 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
60 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
61 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
63 s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
64 t->sw_if_index, t->next_index);
68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
70 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
71 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
72 snat_out2in_worker_handoff_trace_t * t =
73 va_arg (*args, snat_out2in_worker_handoff_trace_t *);
76 m = t->do_handoff ? "next worker" : "same worker";
77 s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
86 } nat44_out2in_reass_trace_t;
88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
90 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
91 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
92 nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
94 s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
95 t->sw_if_index, t->next_index,
96 t->cached ? "cached" : "translated");
101 vlib_node_registration_t snat_out2in_node;
102 vlib_node_registration_t snat_out2in_fast_node;
103 vlib_node_registration_t snat_out2in_worker_handoff_node;
104 vlib_node_registration_t snat_det_out2in_node;
105 vlib_node_registration_t nat44_out2in_reass_node;
107 #define foreach_snat_out2in_error \
108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
109 _(OUT2IN_PACKETS, "Good out2in packets processed") \
110 _(OUT_OF_PORTS, "Out of ports") \
111 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
112 _(NO_TRANSLATION, "No translation") \
113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
114 _(DROP_FRAGMENT, "Drop fragment") \
115 _(MAX_REASS, "Maximum reassemblies exceeded") \
116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
120 foreach_snat_out2in_error
123 } snat_out2in_error_t;
125 static char * snat_out2in_error_strings[] = {
126 #define _(sym,string) string,
127 foreach_snat_out2in_error
132 SNAT_OUT2IN_NEXT_DROP,
133 SNAT_OUT2IN_NEXT_LOOKUP,
134 SNAT_OUT2IN_NEXT_ICMP_ERROR,
135 SNAT_OUT2IN_NEXT_REASS,
136 SNAT_OUT2IN_NEXT_IN2OUT,
138 } snat_out2in_next_t;
141 * @brief Create session for static mapping.
143 * Create NAT session initiated by host from external network with static
146 * @param sm NAT main.
147 * @param b0 Vlib buffer.
148 * @param in2out In2out NAT44 session key.
149 * @param out2in Out2in NAT44 session key.
150 * @param node Vlib node.
152 * @returns SNAT session if successfully created otherwise 0.
154 static inline snat_session_t *
155 create_session_for_static_mapping (snat_main_t *sm,
157 snat_session_key_t in2out,
158 snat_session_key_t out2in,
159 vlib_node_runtime_t * node,
164 clib_bihash_kv_8_8_t kv0;
168 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
170 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
174 ip0 = vlib_buffer_get_current (b0);
175 udp0 = ip4_next_header (ip0);
177 u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
180 clib_warning ("create NAT user failed");
184 s = nat_session_alloc_or_recycle (sm, u, thread_index);
187 clib_warning ("create NAT session failed");
191 s->outside_address_index = ~0;
192 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
193 s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
194 s->ext_host_port = udp0->src_port;
195 user_session_increment (sm, u, 1 /* static */);
198 s->in2out.protocol = out2in.protocol;
200 /* Add to translation hashes */
201 kv0.key = s->in2out.as_u64;
202 kv0.value = s - sm->per_thread_data[thread_index].sessions;
203 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
205 clib_warning ("in2out key add failed");
207 kv0.key = s->out2in.as_u64;
209 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
211 clib_warning ("out2in key add failed");
214 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
215 s->out2in.addr.as_u32,
219 s->in2out.fib_index);
224 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
225 snat_session_key_t *p_key0)
227 icmp46_header_t *icmp0;
228 snat_session_key_t key0;
229 icmp_echo_header_t *echo0, *inner_echo0 = 0;
230 ip4_header_t *inner_ip0;
232 icmp46_header_t *inner_icmp0;
234 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
235 echo0 = (icmp_echo_header_t *)(icmp0+1);
237 if (!icmp_is_error_message (icmp0))
239 key0.protocol = SNAT_PROTOCOL_ICMP;
240 key0.addr = ip0->dst_address;
241 key0.port = echo0->identifier;
245 inner_ip0 = (ip4_header_t *)(echo0+1);
246 l4_header = ip4_next_header (inner_ip0);
247 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
248 key0.addr = inner_ip0->src_address;
249 switch (key0.protocol)
251 case SNAT_PROTOCOL_ICMP:
252 inner_icmp0 = (icmp46_header_t*)l4_header;
253 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
254 key0.port = inner_echo0->identifier;
256 case SNAT_PROTOCOL_UDP:
257 case SNAT_PROTOCOL_TCP:
258 key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
261 return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
265 return -1; /* success */
268 static_always_inline int
269 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
271 icmp46_header_t *icmp0;
272 nat_ed_ses_key_t key0;
273 icmp_echo_header_t *echo0, *inner_echo0 = 0;
274 ip4_header_t *inner_ip0;
276 icmp46_header_t *inner_icmp0;
278 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
279 echo0 = (icmp_echo_header_t *)(icmp0+1);
281 if (!icmp_is_error_message (icmp0))
283 key0.proto = IP_PROTOCOL_ICMP;
284 key0.l_addr = ip0->dst_address;
285 key0.r_addr = ip0->src_address;
286 key0.l_port = key0.r_port = echo0->identifier;
290 inner_ip0 = (ip4_header_t *)(echo0+1);
291 l4_header = ip4_next_header (inner_ip0);
292 key0.proto = inner_ip0->protocol;
293 key0.l_addr = inner_ip0->src_address;
294 key0.r_addr = inner_ip0->dst_address;
295 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
297 case SNAT_PROTOCOL_ICMP:
298 inner_icmp0 = (icmp46_header_t*)l4_header;
299 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
300 key0.l_port = key0.r_port = inner_echo0->identifier;
302 case SNAT_PROTOCOL_UDP:
303 case SNAT_PROTOCOL_TCP:
304 key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
305 key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
316 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
319 snat_session_key_t key;
320 clib_bihash_kv_8_8_t kv, value;
322 key.addr = ip->src_address;
324 key.protocol = proto;
325 key.fib_index = sm->inside_fib_index;
328 if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
336 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
339 nat_ed_ses_key_t key;
340 clib_bihash_kv_16_8_t kv, value;
343 snat_session_t *s = 0;
344 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
346 if (ip->protocol == IP_PROTOCOL_ICMP)
348 if (icmp_get_ed_key (ip, &key))
351 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
353 udp = ip4_next_header(ip);
354 key.r_addr = ip->src_address;
355 key.l_addr = ip->dst_address;
356 key.proto = ip->protocol;
357 key.l_port = udp->dst_port;
358 key.r_port = udp->src_port;
362 key.r_addr = ip->src_address;
363 key.l_addr = ip->dst_address;
364 key.proto = ip->protocol;
365 key.l_port = key.r_port = 0;
368 kv.key[0] = key.as_u64[0];
369 kv.key[1] = key.as_u64[1];
371 if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
373 s = pool_elt_at_index (tsm->sessions, value.value);
377 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
380 u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index, thread_index);
383 clib_warning ("create NAT user failed");
387 s = nat_session_alloc_or_recycle (sm, u, thread_index);
390 clib_warning ("create NAT session failed");
394 s->ext_host_addr = key.r_addr;
395 s->ext_host_port = key.r_port;
396 s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
397 s->outside_address_index = ~0;
398 s->out2in.addr = key.l_addr;
399 s->out2in.port = key.l_port;
400 s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
401 s->out2in.fib_index = 0;
402 s->in2out = s->out2in;
403 user_session_increment (sm, u, 0);
405 kv.value = s - tsm->sessions;
406 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
407 clib_warning ("in2out_ed key add failed");
410 if (ip->protocol == IP_PROTOCOL_TCP)
412 tcp_header_t *tcp = ip4_next_header(ip);
413 if (nat44_set_tcp_session_state (sm, s, tcp, thread_index))
416 /* Per-user LRU list maintenance */
417 clib_dlist_remove (tsm->list_pool, s->per_user_index);
418 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
423 * Get address and port values to be used for ICMP packet translation
424 * and create session if needed
426 * @param[in,out] sm NAT main
427 * @param[in,out] node NAT node runtime
428 * @param[in] thread_index thread index
429 * @param[in,out] b0 buffer containing packet to be translated
430 * @param[out] p_proto protocol used for matching
431 * @param[out] p_value address and port after NAT translation
432 * @param[out] p_dont_translate if packet should not be translated
433 * @param d optional parameter
434 * @param e optional parameter
436 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
437 u32 thread_index, vlib_buffer_t *b0,
438 ip4_header_t *ip0, u8 *p_proto,
439 snat_session_key_t *p_value,
440 u8 *p_dont_translate, void *d, void *e)
442 icmp46_header_t *icmp0;
445 snat_session_key_t key0;
446 snat_session_key_t sm0;
447 snat_session_t *s0 = 0;
448 u8 dont_translate = 0;
449 clib_bihash_kv_8_8_t kv0, value0;
454 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
455 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
456 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
460 err = icmp_get_key (ip0, &key0);
463 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
464 next0 = SNAT_OUT2IN_NEXT_DROP;
467 key0.fib_index = rx_fib_index0;
469 kv0.key = key0.as_u64;
471 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
474 /* Try to match static mapping by external address and port,
475 destination address and port in packet */
476 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
478 if (!sm->forwarding_enabled)
480 /* Don't NAT packet aimed at the intfc address */
481 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
482 ip0->dst_address.as_u32)))
487 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
488 next0 = SNAT_OUT2IN_NEXT_DROP;
494 if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
496 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
499 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
504 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
505 (icmp0->type != ICMP4_echo_request || !is_addr_only)))
507 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
508 next0 = SNAT_OUT2IN_NEXT_DROP;
512 /* Create session initiated by host from external network */
513 s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
518 next0 = SNAT_OUT2IN_NEXT_DROP;
524 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
525 icmp0->type != ICMP4_echo_request &&
526 !icmp_is_error_message (icmp0)))
528 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
529 next0 = SNAT_OUT2IN_NEXT_DROP;
533 if (PREDICT_FALSE (value0.value == ~0ULL))
535 nat_ed_ses_key_t key;
536 clib_bihash_kv_16_8_t s_kv, s_value;
540 if (icmp_get_ed_key (ip0, &key))
542 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
543 next0 = SNAT_OUT2IN_NEXT_DROP;
546 key.fib_index = rx_fib_index0;
547 s_kv.key[0] = key.as_u64[0];
548 s_kv.key[1] = key.as_u64[1];
549 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
550 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
554 next0 = SNAT_OUT2IN_NEXT_DROP;
559 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
564 *p_proto = key0.protocol;
566 *p_value = s0->in2out;
567 *p_dont_translate = dont_translate;
569 *(snat_session_t**)d = s0;
574 * Get address and port values to be used for ICMP packet translation
576 * @param[in] sm NAT main
577 * @param[in,out] node NAT node runtime
578 * @param[in] thread_index thread index
579 * @param[in,out] b0 buffer containing packet to be translated
580 * @param[out] p_proto protocol used for matching
581 * @param[out] p_value address and port after NAT translation
582 * @param[out] p_dont_translate if packet should not be translated
583 * @param d optional parameter
584 * @param e optional parameter
586 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
587 u32 thread_index, vlib_buffer_t *b0,
588 ip4_header_t *ip0, u8 *p_proto,
589 snat_session_key_t *p_value,
590 u8 *p_dont_translate, void *d, void *e)
592 icmp46_header_t *icmp0;
595 snat_session_key_t key0;
596 snat_session_key_t sm0;
597 u8 dont_translate = 0;
602 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
603 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
604 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
606 err = icmp_get_key (ip0, &key0);
609 b0->error = node->errors[err];
610 next0 = SNAT_OUT2IN_NEXT_DROP;
613 key0.fib_index = rx_fib_index0;
615 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
617 /* Don't NAT packet aimed at the intfc address */
618 if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
623 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
624 next0 = SNAT_OUT2IN_NEXT_DROP;
628 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
629 (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
630 !icmp_is_error_message (icmp0)))
632 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
633 next0 = SNAT_OUT2IN_NEXT_DROP;
640 *p_proto = key0.protocol;
641 *p_dont_translate = dont_translate;
645 static inline u32 icmp_out2in (snat_main_t *sm,
648 icmp46_header_t * icmp0,
651 vlib_node_runtime_t * node,
657 snat_session_key_t sm0;
659 icmp_echo_header_t *echo0, *inner_echo0 = 0;
660 ip4_header_t *inner_ip0 = 0;
662 icmp46_header_t *inner_icmp0;
664 u32 new_addr0, old_addr0;
665 u16 old_id0, new_id0;
670 echo0 = (icmp_echo_header_t *)(icmp0+1);
672 next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
673 &protocol, &sm0, &dont_translate, d, e);
676 if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
679 sum0 = ip_incremental_checksum (0, icmp0,
680 ntohs(ip0->length) - ip4_header_bytes (ip0));
681 checksum0 = ~ip_csum_fold (sum0);
682 if (checksum0 != 0 && checksum0 != 0xffff)
684 next0 = SNAT_OUT2IN_NEXT_DROP;
688 old_addr0 = ip0->dst_address.as_u32;
689 new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
690 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
692 sum0 = ip0->checksum;
693 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
694 dst_address /* changed member */);
695 ip0->checksum = ip_csum_fold (sum0);
697 if (icmp0->checksum == 0)
698 icmp0->checksum = 0xffff;
700 if (!icmp_is_error_message (icmp0))
703 if (PREDICT_FALSE(new_id0 != echo0->identifier))
705 old_id0 = echo0->identifier;
707 echo0->identifier = new_id0;
709 sum0 = icmp0->checksum;
710 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
711 identifier /* changed member */);
712 icmp0->checksum = ip_csum_fold (sum0);
717 inner_ip0 = (ip4_header_t *)(echo0+1);
718 l4_header = ip4_next_header (inner_ip0);
720 if (!ip4_header_checksum_is_valid (inner_ip0))
722 next0 = SNAT_OUT2IN_NEXT_DROP;
726 old_addr0 = inner_ip0->src_address.as_u32;
727 inner_ip0->src_address = sm0.addr;
728 new_addr0 = inner_ip0->src_address.as_u32;
730 sum0 = icmp0->checksum;
731 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
732 src_address /* changed member */);
733 icmp0->checksum = ip_csum_fold (sum0);
737 case SNAT_PROTOCOL_ICMP:
738 inner_icmp0 = (icmp46_header_t*)l4_header;
739 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
741 old_id0 = inner_echo0->identifier;
743 inner_echo0->identifier = new_id0;
745 sum0 = icmp0->checksum;
746 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
748 icmp0->checksum = ip_csum_fold (sum0);
750 case SNAT_PROTOCOL_UDP:
751 case SNAT_PROTOCOL_TCP:
752 old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
754 ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
756 sum0 = icmp0->checksum;
757 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
759 icmp0->checksum = ip_csum_fold (sum0);
771 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
774 icmp46_header_t * icmp0,
777 vlib_node_runtime_t * node,
780 snat_session_t ** p_s0)
782 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
783 next0, thread_index, p_s0, 0);
784 snat_session_t * s0 = *p_s0;
785 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
788 s0->last_heard = now;
790 s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
791 /* Per-user LRU list maintenance */
792 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
794 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
795 s0->per_user_list_head_index,
801 static snat_session_t *
802 snat_out2in_unknown_proto (snat_main_t *sm,
809 vlib_node_runtime_t * node)
811 clib_bihash_kv_8_8_t kv, value;
812 clib_bihash_kv_16_8_t s_kv, s_value;
813 snat_static_mapping_t *m;
814 snat_session_key_t m_key;
815 u32 old_addr, new_addr;
817 nat_ed_ses_key_t key;
819 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
822 old_addr = ip->dst_address.as_u32;
824 key.l_addr = ip->dst_address;
825 key.r_addr = ip->src_address;
826 key.fib_index = rx_fib_index;
827 key.proto = ip->protocol;
830 s_kv.key[0] = key.as_u64[0];
831 s_kv.key[1] = key.as_u64[1];
833 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
835 s = pool_elt_at_index (tsm->sessions, s_value.value);
836 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
840 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
842 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
846 m_key.addr = ip->dst_address;
849 m_key.fib_index = rx_fib_index;
850 kv.key = m_key.as_u64;
851 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
853 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
857 m = pool_elt_at_index (sm->static_mappings, value.value);
859 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
861 u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
865 clib_warning ("create NAT user failed");
869 /* Create a new session */
870 s = nat_session_alloc_or_recycle (sm, u, thread_index);
873 clib_warning ("create NAT session failed");
877 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
878 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
879 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
880 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
881 s->outside_address_index = ~0;
882 s->out2in.addr.as_u32 = old_addr;
883 s->out2in.fib_index = rx_fib_index;
884 s->in2out.addr.as_u32 = new_addr;
885 s->in2out.fib_index = m->fib_index;
886 s->in2out.port = s->out2in.port = ip->protocol;
887 user_session_increment (sm, u, 1 /* static */);
889 /* Add to lookup tables */
890 s_kv.value = s - tsm->sessions;
891 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
892 clib_warning ("out2in key add failed");
894 key.l_addr = ip->dst_address;
895 key.fib_index = m->fib_index;
896 s_kv.key[0] = key.as_u64[0];
897 s_kv.key[1] = key.as_u64[1];
898 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
899 clib_warning ("in2out key add failed");
902 /* Update IP checksum */
904 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
905 ip->checksum = ip_csum_fold (sum);
907 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
912 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
913 /* Per-user LRU list maintenance */
914 clib_dlist_remove (tsm->list_pool, s->per_user_index);
915 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
921 static snat_session_t *
922 snat_out2in_lb (snat_main_t *sm,
929 vlib_node_runtime_t * node)
931 nat_ed_ses_key_t key;
932 clib_bihash_kv_16_8_t s_kv, s_value;
933 udp_header_t *udp = ip4_next_header (ip);
934 tcp_header_t *tcp = (tcp_header_t *) udp;
935 snat_session_t *s = 0;
936 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
937 snat_session_key_t e_key, l_key;
938 u32 old_addr, new_addr;
939 u32 proto = ip_proto_to_snat_proto (ip->protocol);
940 u16 new_port, old_port;
944 snat_session_key_t eh_key;
945 twice_nat_type_t twice_nat;
948 old_addr = ip->dst_address.as_u32;
950 key.l_addr = ip->dst_address;
951 key.r_addr = ip->src_address;
952 key.fib_index = rx_fib_index;
953 key.proto = ip->protocol;
954 key.r_port = udp->src_port;
955 key.l_port = udp->dst_port;
956 s_kv.key[0] = key.as_u64[0];
957 s_kv.key[1] = key.as_u64[1];
959 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
961 s = pool_elt_at_index (tsm->sessions, s_value.value);
965 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
967 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
971 e_key.addr = ip->dst_address;
972 e_key.port = udp->dst_port;
973 e_key.protocol = proto;
974 e_key.fib_index = rx_fib_index;
975 if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat, &lb))
978 u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
982 clib_warning ("create NAT user failed");
986 s = nat_session_alloc_or_recycle (sm, u, thread_index);
989 clib_warning ("create NAT session failed");
993 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
994 s->ext_host_port = udp->src_port;
995 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
997 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
998 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
999 s->outside_address_index = ~0;
1002 user_session_increment (sm, u, 1 /* static */);
1004 /* Add to lookup tables */
1005 s_kv.value = s - tsm->sessions;
1006 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1007 clib_warning ("out2in-ed key add failed");
1009 if (twice_nat == TWICE_NAT ||
1010 (twice_nat == TWICE_NAT_SELF &&
1011 ip->src_address.as_u32 == l_key.addr.as_u32))
1013 eh_key.protocol = proto;
1014 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1015 thread_index, &eh_key,
1017 sm->port_per_thread,
1018 sm->per_thread_data[thread_index].snat_thread_index))
1020 b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1023 key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1024 key.r_port = s->ext_host_nat_port = eh_key.port;
1025 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1027 key.l_addr = l_key.addr;
1028 key.fib_index = l_key.fib_index;
1029 key.l_port = l_key.port;
1030 s_kv.key[0] = key.as_u64[0];
1031 s_kv.key[1] = key.as_u64[1];
1032 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1033 clib_warning ("in2out-ed key add failed");
1036 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1038 /* Update IP checksum */
1040 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1041 if (is_twice_nat_session (s))
1042 sum = ip_csum_update (sum, ip->src_address.as_u32,
1043 s->ext_host_nat_addr.as_u32, ip4_header_t,
1045 ip->checksum = ip_csum_fold (sum);
1047 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1049 if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1051 old_port = tcp->dst_port;
1052 tcp->dst_port = s->in2out.port;
1053 new_port = tcp->dst_port;
1055 sum = tcp->checksum;
1056 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1057 sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1058 if (is_twice_nat_session (s))
1060 sum = ip_csum_update (sum, ip->src_address.as_u32,
1061 s->ext_host_nat_addr.as_u32, ip4_header_t,
1063 sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1064 ip4_header_t, length);
1065 tcp->src_port = s->ext_host_nat_port;
1066 ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1068 tcp->checksum = ip_csum_fold(sum);
1069 if (nat44_set_tcp_session_state (sm, s, tcp, thread_index))
1074 udp->dst_port = s->in2out.port;
1075 if (is_twice_nat_session (s))
1077 udp->src_port = s->ext_host_nat_port;
1078 ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1084 s->last_heard = now;
1086 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1087 /* Per-user LRU list maintenance */
1088 clib_dlist_remove (tsm->list_pool, s->per_user_index);
1089 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1096 snat_out2in_node_fn (vlib_main_t * vm,
1097 vlib_node_runtime_t * node,
1098 vlib_frame_t * frame)
1100 u32 n_left_from, * from, * to_next;
1101 snat_out2in_next_t next_index;
1102 u32 pkts_processed = 0;
1103 snat_main_t * sm = &snat_main;
1104 f64 now = vlib_time_now (vm);
1105 u32 thread_index = vlib_get_thread_index ();
1107 from = vlib_frame_vector_args (frame);
1108 n_left_from = frame->n_vectors;
1109 next_index = node->cached_next_index;
1111 while (n_left_from > 0)
1115 vlib_get_next_frame (vm, node, next_index,
1116 to_next, n_left_to_next);
1118 while (n_left_from >= 4 && n_left_to_next >= 2)
1121 vlib_buffer_t * b0, * b1;
1122 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1123 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1124 u32 sw_if_index0, sw_if_index1;
1125 ip4_header_t * ip0, *ip1;
1126 ip_csum_t sum0, sum1;
1127 u32 new_addr0, old_addr0;
1128 u16 new_port0, old_port0;
1129 u32 new_addr1, old_addr1;
1130 u16 new_port1, old_port1;
1131 udp_header_t * udp0, * udp1;
1132 tcp_header_t * tcp0, * tcp1;
1133 icmp46_header_t * icmp0, * icmp1;
1134 snat_session_key_t key0, key1, sm0, sm1;
1135 u32 rx_fib_index0, rx_fib_index1;
1137 snat_session_t * s0 = 0, * s1 = 0;
1138 clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1140 /* Prefetch next iteration. */
1142 vlib_buffer_t * p2, * p3;
1144 p2 = vlib_get_buffer (vm, from[2]);
1145 p3 = vlib_get_buffer (vm, from[3]);
1147 vlib_prefetch_buffer_header (p2, LOAD);
1148 vlib_prefetch_buffer_header (p3, LOAD);
1150 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1151 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1154 /* speculatively enqueue b0 and b1 to the current next frame */
1155 to_next[0] = bi0 = from[0];
1156 to_next[1] = bi1 = from[1];
1160 n_left_to_next -= 2;
1162 b0 = vlib_get_buffer (vm, bi0);
1163 b1 = vlib_get_buffer (vm, bi1);
1165 vnet_buffer (b0)->snat.flags = 0;
1166 vnet_buffer (b1)->snat.flags = 0;
1168 ip0 = vlib_buffer_get_current (b0);
1169 udp0 = ip4_next_header (ip0);
1170 tcp0 = (tcp_header_t *) udp0;
1171 icmp0 = (icmp46_header_t *) udp0;
1173 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1174 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1177 if (PREDICT_FALSE(ip0->ttl == 1))
1179 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1180 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1181 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1183 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1187 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1189 if (PREDICT_FALSE (proto0 == ~0))
1191 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1192 thread_index, now, vm, node);
1193 if (!sm->forwarding_enabled)
1195 next0 = SNAT_OUT2IN_NEXT_DROP;
1199 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1201 next0 = icmp_out2in_slow_path
1202 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1203 next0, now, thread_index, &s0);
1207 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1209 next0 = SNAT_OUT2IN_NEXT_REASS;
1213 key0.addr = ip0->dst_address;
1214 key0.port = udp0->dst_port;
1215 key0.protocol = proto0;
1216 key0.fib_index = rx_fib_index0;
1218 kv0.key = key0.as_u64;
1220 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1223 /* Try to match static mapping by external address and port,
1224 destination address and port in packet */
1225 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1228 * Send DHCP packets to the ipv4 stack, or we won't
1229 * be able to use dhcp client on the outside interface
1231 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1232 && (udp0->dst_port ==
1233 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1236 (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1240 if (!sm->forwarding_enabled)
1242 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1243 next0 = SNAT_OUT2IN_NEXT_DROP;
1248 if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1250 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1253 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1258 /* Create session initiated by host from external network */
1259 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1263 next0 = SNAT_OUT2IN_NEXT_DROP;
1269 if (PREDICT_FALSE (value0.value == ~0ULL))
1271 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1274 next0 = SNAT_OUT2IN_NEXT_DROP;
1279 s0 = pool_elt_at_index (
1280 sm->per_thread_data[thread_index].sessions,
1285 old_addr0 = ip0->dst_address.as_u32;
1286 ip0->dst_address = s0->in2out.addr;
1287 new_addr0 = ip0->dst_address.as_u32;
1288 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1290 sum0 = ip0->checksum;
1291 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1293 dst_address /* changed member */);
1294 ip0->checksum = ip_csum_fold (sum0);
1296 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1298 old_port0 = tcp0->dst_port;
1299 tcp0->dst_port = s0->in2out.port;
1300 new_port0 = tcp0->dst_port;
1302 sum0 = tcp0->checksum;
1303 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1305 dst_address /* changed member */);
1307 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1308 ip4_header_t /* cheat */,
1309 length /* changed member */);
1310 tcp0->checksum = ip_csum_fold(sum0);
1311 if (nat44_set_tcp_session_state (sm, s0, tcp0, thread_index))
1316 old_port0 = udp0->dst_port;
1317 udp0->dst_port = s0->in2out.port;
1322 s0->last_heard = now;
1324 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1325 /* Per-user LRU list maintenance */
1326 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1327 s0->per_user_index);
1328 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1329 s0->per_user_list_head_index,
1330 s0->per_user_index);
1333 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1334 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1336 snat_out2in_trace_t *t =
1337 vlib_add_trace (vm, node, b0, sizeof (*t));
1338 t->sw_if_index = sw_if_index0;
1339 t->next_index = next0;
1340 t->session_index = ~0;
1342 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1345 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1348 ip1 = vlib_buffer_get_current (b1);
1349 udp1 = ip4_next_header (ip1);
1350 tcp1 = (tcp_header_t *) udp1;
1351 icmp1 = (icmp46_header_t *) udp1;
1353 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1354 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1357 if (PREDICT_FALSE(ip1->ttl == 1))
1359 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1360 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1361 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1363 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1367 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1369 if (PREDICT_FALSE (proto1 == ~0))
1371 s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1372 thread_index, now, vm, node);
1373 if (!sm->forwarding_enabled)
1375 next1 = SNAT_OUT2IN_NEXT_DROP;
1379 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1381 next1 = icmp_out2in_slow_path
1382 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1383 next1, now, thread_index, &s1);
1387 if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1389 next1 = SNAT_OUT2IN_NEXT_REASS;
1393 key1.addr = ip1->dst_address;
1394 key1.port = udp1->dst_port;
1395 key1.protocol = proto1;
1396 key1.fib_index = rx_fib_index1;
1398 kv1.key = key1.as_u64;
1400 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1403 /* Try to match static mapping by external address and port,
1404 destination address and port in packet */
1405 if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
1408 * Send DHCP packets to the ipv4 stack, or we won't
1409 * be able to use dhcp client on the outside interface
1411 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1412 && (udp1->dst_port ==
1413 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1416 (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
1420 if (!sm->forwarding_enabled)
1422 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1423 next1 = SNAT_OUT2IN_NEXT_DROP;
1428 if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1430 next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1433 create_bypass_for_fwd(sm, ip1, rx_fib_index1, thread_index);
1438 /* Create session initiated by host from external network */
1439 s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1443 next1 = SNAT_OUT2IN_NEXT_DROP;
1449 if (PREDICT_FALSE (value1.value == ~0ULL))
1451 s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1454 next1 = SNAT_OUT2IN_NEXT_DROP;
1459 s1 = pool_elt_at_index (
1460 sm->per_thread_data[thread_index].sessions,
1465 old_addr1 = ip1->dst_address.as_u32;
1466 ip1->dst_address = s1->in2out.addr;
1467 new_addr1 = ip1->dst_address.as_u32;
1468 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1470 sum1 = ip1->checksum;
1471 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1473 dst_address /* changed member */);
1474 ip1->checksum = ip_csum_fold (sum1);
1476 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1478 old_port1 = tcp1->dst_port;
1479 tcp1->dst_port = s1->in2out.port;
1480 new_port1 = tcp1->dst_port;
1482 sum1 = tcp1->checksum;
1483 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1485 dst_address /* changed member */);
1487 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1488 ip4_header_t /* cheat */,
1489 length /* changed member */);
1490 tcp1->checksum = ip_csum_fold(sum1);
1491 if (nat44_set_tcp_session_state (sm, s1, tcp1, thread_index))
1496 old_port1 = udp1->dst_port;
1497 udp1->dst_port = s1->in2out.port;
1502 s1->last_heard = now;
1504 s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1505 /* Per-user LRU list maintenance */
1506 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1507 s1->per_user_index);
1508 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1509 s1->per_user_list_head_index,
1510 s1->per_user_index);
1513 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1514 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1516 snat_out2in_trace_t *t =
1517 vlib_add_trace (vm, node, b1, sizeof (*t));
1518 t->sw_if_index = sw_if_index1;
1519 t->next_index = next1;
1520 t->session_index = ~0;
1522 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1525 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1527 /* verify speculative enqueues, maybe switch current next frame */
1528 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1529 to_next, n_left_to_next,
1530 bi0, bi1, next0, next1);
1533 while (n_left_from > 0 && n_left_to_next > 0)
1537 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1541 u32 new_addr0, old_addr0;
1542 u16 new_port0, old_port0;
1543 udp_header_t * udp0;
1544 tcp_header_t * tcp0;
1545 icmp46_header_t * icmp0;
1546 snat_session_key_t key0, sm0;
1549 snat_session_t * s0 = 0;
1550 clib_bihash_kv_8_8_t kv0, value0;
1552 /* speculatively enqueue b0 to the current next frame */
1558 n_left_to_next -= 1;
1560 b0 = vlib_get_buffer (vm, bi0);
1562 vnet_buffer (b0)->snat.flags = 0;
1564 ip0 = vlib_buffer_get_current (b0);
1565 udp0 = ip4_next_header (ip0);
1566 tcp0 = (tcp_header_t *) udp0;
1567 icmp0 = (icmp46_header_t *) udp0;
1569 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1570 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1573 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1575 if (PREDICT_FALSE (proto0 == ~0))
1577 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1578 thread_index, now, vm, node);
1579 if (!sm->forwarding_enabled)
1581 next0 = SNAT_OUT2IN_NEXT_DROP;
1585 if (PREDICT_FALSE(ip0->ttl == 1))
1587 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1588 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1589 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1591 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1595 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1597 next0 = icmp_out2in_slow_path
1598 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1599 next0, now, thread_index, &s0);
1603 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1605 next0 = SNAT_OUT2IN_NEXT_REASS;
1609 key0.addr = ip0->dst_address;
1610 key0.port = udp0->dst_port;
1611 key0.protocol = proto0;
1612 key0.fib_index = rx_fib_index0;
1614 kv0.key = key0.as_u64;
1616 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1619 /* Try to match static mapping by external address and port,
1620 destination address and port in packet */
1621 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1624 * Send DHCP packets to the ipv4 stack, or we won't
1625 * be able to use dhcp client on the outside interface
1627 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1628 && (udp0->dst_port ==
1629 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1632 (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1636 if (!sm->forwarding_enabled)
1638 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1639 next0 = SNAT_OUT2IN_NEXT_DROP;
1644 if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1646 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1649 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1654 /* Create session initiated by host from external network */
1655 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1659 next0 = SNAT_OUT2IN_NEXT_DROP;
1665 if (PREDICT_FALSE (value0.value == ~0ULL))
1667 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1670 next0 = SNAT_OUT2IN_NEXT_DROP;
1675 s0 = pool_elt_at_index (
1676 sm->per_thread_data[thread_index].sessions,
1681 old_addr0 = ip0->dst_address.as_u32;
1682 ip0->dst_address = s0->in2out.addr;
1683 new_addr0 = ip0->dst_address.as_u32;
1684 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1686 sum0 = ip0->checksum;
1687 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1689 dst_address /* changed member */);
1690 ip0->checksum = ip_csum_fold (sum0);
1692 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1694 old_port0 = tcp0->dst_port;
1695 tcp0->dst_port = s0->in2out.port;
1696 new_port0 = tcp0->dst_port;
1698 sum0 = tcp0->checksum;
1699 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1701 dst_address /* changed member */);
1703 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1704 ip4_header_t /* cheat */,
1705 length /* changed member */);
1706 tcp0->checksum = ip_csum_fold(sum0);
1707 if (nat44_set_tcp_session_state (sm, s0, tcp0, thread_index))
1712 old_port0 = udp0->dst_port;
1713 udp0->dst_port = s0->in2out.port;
1718 s0->last_heard = now;
1720 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1721 /* Per-user LRU list maintenance */
1722 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1723 s0->per_user_index);
1724 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1725 s0->per_user_list_head_index,
1726 s0->per_user_index);
1729 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1730 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1732 snat_out2in_trace_t *t =
1733 vlib_add_trace (vm, node, b0, sizeof (*t));
1734 t->sw_if_index = sw_if_index0;
1735 t->next_index = next0;
1736 t->session_index = ~0;
1738 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1741 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1743 /* verify speculative enqueue, maybe switch current next frame */
1744 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1745 to_next, n_left_to_next,
1749 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1752 vlib_node_increment_counter (vm, snat_out2in_node.index,
1753 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1755 return frame->n_vectors;
1758 VLIB_REGISTER_NODE (snat_out2in_node) = {
1759 .function = snat_out2in_node_fn,
1760 .name = "nat44-out2in",
1761 .vector_size = sizeof (u32),
1762 .format_trace = format_snat_out2in_trace,
1763 .type = VLIB_NODE_TYPE_INTERNAL,
1765 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1766 .error_strings = snat_out2in_error_strings,
1768 .runtime_data_bytes = sizeof (snat_runtime_t),
1770 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1772 /* edit / add dispositions here */
1774 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1775 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1776 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1777 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1778 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1781 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1784 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1785 vlib_node_runtime_t * node,
1786 vlib_frame_t * frame)
1788 u32 n_left_from, *from, *to_next;
1789 snat_out2in_next_t next_index;
1790 u32 pkts_processed = 0;
1791 snat_main_t *sm = &snat_main;
1792 f64 now = vlib_time_now (vm);
1793 u32 thread_index = vlib_get_thread_index ();
1794 snat_main_per_thread_data_t *per_thread_data =
1795 &sm->per_thread_data[thread_index];
1796 u32 *fragments_to_drop = 0;
1797 u32 *fragments_to_loopback = 0;
1799 from = vlib_frame_vector_args (frame);
1800 n_left_from = frame->n_vectors;
1801 next_index = node->cached_next_index;
1803 while (n_left_from > 0)
1807 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1809 while (n_left_from > 0 && n_left_to_next > 0)
1811 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1816 nat_reass_ip4_t *reass0;
1817 udp_header_t * udp0;
1818 tcp_header_t * tcp0;
1819 snat_session_key_t key0, sm0;
1820 clib_bihash_kv_8_8_t kv0, value0;
1821 snat_session_t * s0 = 0;
1822 u16 old_port0, new_port0;
1825 /* speculatively enqueue b0 to the current next frame */
1831 n_left_to_next -= 1;
1833 b0 = vlib_get_buffer (vm, bi0);
1834 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1836 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1837 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1840 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1842 next0 = SNAT_OUT2IN_NEXT_DROP;
1843 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1847 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1848 udp0 = ip4_next_header (ip0);
1849 tcp0 = (tcp_header_t *) udp0;
1850 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1852 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1857 &fragments_to_drop);
1859 if (PREDICT_FALSE (!reass0))
1861 next0 = SNAT_OUT2IN_NEXT_DROP;
1862 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1866 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1868 key0.addr = ip0->dst_address;
1869 key0.port = udp0->dst_port;
1870 key0.protocol = proto0;
1871 key0.fib_index = rx_fib_index0;
1872 kv0.key = key0.as_u64;
1874 if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1876 /* Try to match static mapping by external address and port,
1877 destination address and port in packet */
1878 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1881 * Send DHCP packets to the ipv4 stack, or we won't
1882 * be able to use dhcp client on the outside interface
1884 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1886 == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1889 (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1894 if (!sm->forwarding_enabled)
1896 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1897 next0 = SNAT_OUT2IN_NEXT_DROP;
1902 if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1904 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1907 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1912 /* Create session initiated by host from external network */
1913 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1917 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1918 next0 = SNAT_OUT2IN_NEXT_DROP;
1921 reass0->sess_index = s0 - per_thread_data->sessions;
1922 reass0->thread_index = thread_index;
1926 s0 = pool_elt_at_index (per_thread_data->sessions,
1928 reass0->sess_index = value0.value;
1930 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1934 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1936 if (nat_ip4_reass_add_fragment (reass0, bi0))
1938 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1939 next0 = SNAT_OUT2IN_NEXT_DROP;
1945 s0 = pool_elt_at_index (per_thread_data->sessions,
1946 reass0->sess_index);
1949 old_addr0 = ip0->dst_address.as_u32;
1950 ip0->dst_address = s0->in2out.addr;
1951 new_addr0 = ip0->dst_address.as_u32;
1952 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1954 sum0 = ip0->checksum;
1955 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1957 dst_address /* changed member */);
1958 ip0->checksum = ip_csum_fold (sum0);
1960 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1962 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1964 old_port0 = tcp0->dst_port;
1965 tcp0->dst_port = s0->in2out.port;
1966 new_port0 = tcp0->dst_port;
1968 sum0 = tcp0->checksum;
1969 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1971 dst_address /* changed member */);
1973 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1974 ip4_header_t /* cheat */,
1975 length /* changed member */);
1976 tcp0->checksum = ip_csum_fold(sum0);
1977 if (nat44_set_tcp_session_state (sm, s0, tcp0, thread_index))
1982 old_port0 = udp0->dst_port;
1983 udp0->dst_port = s0->in2out.port;
1989 s0->last_heard = now;
1991 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1992 /* Per-user LRU list maintenance */
1993 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1994 s0->per_user_index);
1995 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1996 s0->per_user_list_head_index,
1997 s0->per_user_index);
2000 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2001 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2003 nat44_out2in_reass_trace_t *t =
2004 vlib_add_trace (vm, node, b0, sizeof (*t));
2005 t->cached = cached0;
2006 t->sw_if_index = sw_if_index0;
2007 t->next_index = next0;
2017 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2019 /* verify speculative enqueue, maybe switch current next frame */
2020 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2021 to_next, n_left_to_next,
2025 if (n_left_from == 0 && vec_len (fragments_to_loopback))
2027 from = vlib_frame_vector_args (frame);
2028 u32 len = vec_len (fragments_to_loopback);
2029 if (len <= VLIB_FRAME_SIZE)
2031 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2033 vec_reset_length (fragments_to_loopback);
2038 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2039 sizeof (u32) * VLIB_FRAME_SIZE);
2040 n_left_from = VLIB_FRAME_SIZE;
2041 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2046 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2049 vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
2050 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2053 nat_send_all_to_node (vm, fragments_to_drop, node,
2054 &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
2055 SNAT_OUT2IN_NEXT_DROP);
2057 vec_free (fragments_to_drop);
2058 vec_free (fragments_to_loopback);
2059 return frame->n_vectors;
2062 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
2063 .function = nat44_out2in_reass_node_fn,
2064 .name = "nat44-out2in-reass",
2065 .vector_size = sizeof (u32),
2066 .format_trace = format_nat44_out2in_reass_trace,
2067 .type = VLIB_NODE_TYPE_INTERNAL,
2069 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2070 .error_strings = snat_out2in_error_strings,
2072 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2074 /* edit / add dispositions here */
2076 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2077 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2078 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2079 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2080 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2083 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2084 nat44_out2in_reass_node_fn);
2086 /**************************/
2087 /*** deterministic mode ***/
2088 /**************************/
2090 snat_det_out2in_node_fn (vlib_main_t * vm,
2091 vlib_node_runtime_t * node,
2092 vlib_frame_t * frame)
2094 u32 n_left_from, * from, * to_next;
2095 snat_out2in_next_t next_index;
2096 u32 pkts_processed = 0;
2097 snat_main_t * sm = &snat_main;
2098 u32 thread_index = vlib_get_thread_index ();
2100 from = vlib_frame_vector_args (frame);
2101 n_left_from = frame->n_vectors;
2102 next_index = node->cached_next_index;
2104 while (n_left_from > 0)
2108 vlib_get_next_frame (vm, node, next_index,
2109 to_next, n_left_to_next);
2111 while (n_left_from >= 4 && n_left_to_next >= 2)
2114 vlib_buffer_t * b0, * b1;
2115 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2116 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2117 u32 sw_if_index0, sw_if_index1;
2118 ip4_header_t * ip0, * ip1;
2119 ip_csum_t sum0, sum1;
2120 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2121 u16 new_port0, old_port0, old_port1, new_port1;
2122 udp_header_t * udp0, * udp1;
2123 tcp_header_t * tcp0, * tcp1;
2125 snat_det_out_key_t key0, key1;
2126 snat_det_map_t * dm0, * dm1;
2127 snat_det_session_t * ses0 = 0, * ses1 = 0;
2128 u32 rx_fib_index0, rx_fib_index1;
2129 icmp46_header_t * icmp0, * icmp1;
2131 /* Prefetch next iteration. */
2133 vlib_buffer_t * p2, * p3;
2135 p2 = vlib_get_buffer (vm, from[2]);
2136 p3 = vlib_get_buffer (vm, from[3]);
2138 vlib_prefetch_buffer_header (p2, LOAD);
2139 vlib_prefetch_buffer_header (p3, LOAD);
2141 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2142 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2145 /* speculatively enqueue b0 and b1 to the current next frame */
2146 to_next[0] = bi0 = from[0];
2147 to_next[1] = bi1 = from[1];
2151 n_left_to_next -= 2;
2153 b0 = vlib_get_buffer (vm, bi0);
2154 b1 = vlib_get_buffer (vm, bi1);
2156 ip0 = vlib_buffer_get_current (b0);
2157 udp0 = ip4_next_header (ip0);
2158 tcp0 = (tcp_header_t *) udp0;
2160 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2162 if (PREDICT_FALSE(ip0->ttl == 1))
2164 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2165 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2166 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2168 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2172 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2174 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2176 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2177 icmp0 = (icmp46_header_t *) udp0;
2179 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2180 rx_fib_index0, node, next0, thread_index,
2185 key0.ext_host_addr = ip0->src_address;
2186 key0.ext_host_port = tcp0->src;
2187 key0.out_port = tcp0->dst;
2189 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2190 if (PREDICT_FALSE(!dm0))
2192 clib_warning("unknown dst address: %U",
2193 format_ip4_address, &ip0->dst_address);
2194 next0 = SNAT_OUT2IN_NEXT_DROP;
2195 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2199 snat_det_reverse(dm0, &ip0->dst_address,
2200 clib_net_to_host_u16(tcp0->dst), &new_addr0);
2202 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2203 if (PREDICT_FALSE(!ses0))
2205 clib_warning("no match src %U:%d dst %U:%d for user %U",
2206 format_ip4_address, &ip0->src_address,
2207 clib_net_to_host_u16 (tcp0->src),
2208 format_ip4_address, &ip0->dst_address,
2209 clib_net_to_host_u16 (tcp0->dst),
2210 format_ip4_address, &new_addr0);
2211 next0 = SNAT_OUT2IN_NEXT_DROP;
2212 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2215 new_port0 = ses0->in_port;
2217 old_addr0 = ip0->dst_address;
2218 ip0->dst_address = new_addr0;
2219 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2221 sum0 = ip0->checksum;
2222 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2224 dst_address /* changed member */);
2225 ip0->checksum = ip_csum_fold (sum0);
2227 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2229 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2230 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2231 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2232 snat_det_ses_close(dm0, ses0);
2234 old_port0 = tcp0->dst;
2235 tcp0->dst = new_port0;
2237 sum0 = tcp0->checksum;
2238 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2240 dst_address /* changed member */);
2242 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2243 ip4_header_t /* cheat */,
2244 length /* changed member */);
2245 tcp0->checksum = ip_csum_fold(sum0);
2249 old_port0 = udp0->dst_port;
2250 udp0->dst_port = new_port0;
2256 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2257 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2259 snat_out2in_trace_t *t =
2260 vlib_add_trace (vm, node, b0, sizeof (*t));
2261 t->sw_if_index = sw_if_index0;
2262 t->next_index = next0;
2263 t->session_index = ~0;
2265 t->session_index = ses0 - dm0->sessions;
2268 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2270 b1 = vlib_get_buffer (vm, bi1);
2272 ip1 = vlib_buffer_get_current (b1);
2273 udp1 = ip4_next_header (ip1);
2274 tcp1 = (tcp_header_t *) udp1;
2276 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2278 if (PREDICT_FALSE(ip1->ttl == 1))
2280 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2281 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2282 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2284 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2288 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2290 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2292 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2293 icmp1 = (icmp46_header_t *) udp1;
2295 next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2296 rx_fib_index1, node, next1, thread_index,
2301 key1.ext_host_addr = ip1->src_address;
2302 key1.ext_host_port = tcp1->src;
2303 key1.out_port = tcp1->dst;
2305 dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2306 if (PREDICT_FALSE(!dm1))
2308 clib_warning("unknown dst address: %U",
2309 format_ip4_address, &ip1->dst_address);
2310 next1 = SNAT_OUT2IN_NEXT_DROP;
2311 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2315 snat_det_reverse(dm1, &ip1->dst_address,
2316 clib_net_to_host_u16(tcp1->dst), &new_addr1);
2318 ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2319 if (PREDICT_FALSE(!ses1))
2321 clib_warning("no match src %U:%d dst %U:%d for user %U",
2322 format_ip4_address, &ip1->src_address,
2323 clib_net_to_host_u16 (tcp1->src),
2324 format_ip4_address, &ip1->dst_address,
2325 clib_net_to_host_u16 (tcp1->dst),
2326 format_ip4_address, &new_addr1);
2327 next1 = SNAT_OUT2IN_NEXT_DROP;
2328 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2331 new_port1 = ses1->in_port;
2333 old_addr1 = ip1->dst_address;
2334 ip1->dst_address = new_addr1;
2335 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2337 sum1 = ip1->checksum;
2338 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2340 dst_address /* changed member */);
2341 ip1->checksum = ip_csum_fold (sum1);
2343 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2345 if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2346 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2347 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2348 snat_det_ses_close(dm1, ses1);
2350 old_port1 = tcp1->dst;
2351 tcp1->dst = new_port1;
2353 sum1 = tcp1->checksum;
2354 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2356 dst_address /* changed member */);
2358 sum1 = ip_csum_update (sum1, old_port1, new_port1,
2359 ip4_header_t /* cheat */,
2360 length /* changed member */);
2361 tcp1->checksum = ip_csum_fold(sum1);
2365 old_port1 = udp1->dst_port;
2366 udp1->dst_port = new_port1;
2372 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2373 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2375 snat_out2in_trace_t *t =
2376 vlib_add_trace (vm, node, b1, sizeof (*t));
2377 t->sw_if_index = sw_if_index1;
2378 t->next_index = next1;
2379 t->session_index = ~0;
2381 t->session_index = ses1 - dm1->sessions;
2384 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2386 /* verify speculative enqueues, maybe switch current next frame */
2387 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2388 to_next, n_left_to_next,
2389 bi0, bi1, next0, next1);
2392 while (n_left_from > 0 && n_left_to_next > 0)
2396 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2400 ip4_address_t new_addr0, old_addr0;
2401 u16 new_port0, old_port0;
2402 udp_header_t * udp0;
2403 tcp_header_t * tcp0;
2405 snat_det_out_key_t key0;
2406 snat_det_map_t * dm0;
2407 snat_det_session_t * ses0 = 0;
2409 icmp46_header_t * icmp0;
2411 /* speculatively enqueue b0 to the current next frame */
2417 n_left_to_next -= 1;
2419 b0 = vlib_get_buffer (vm, bi0);
2421 ip0 = vlib_buffer_get_current (b0);
2422 udp0 = ip4_next_header (ip0);
2423 tcp0 = (tcp_header_t *) udp0;
2425 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2427 if (PREDICT_FALSE(ip0->ttl == 1))
2429 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2430 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2431 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2433 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2437 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2439 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2441 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2442 icmp0 = (icmp46_header_t *) udp0;
2444 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2445 rx_fib_index0, node, next0, thread_index,
2450 key0.ext_host_addr = ip0->src_address;
2451 key0.ext_host_port = tcp0->src;
2452 key0.out_port = tcp0->dst;
2454 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2455 if (PREDICT_FALSE(!dm0))
2457 clib_warning("unknown dst address: %U",
2458 format_ip4_address, &ip0->dst_address);
2459 next0 = SNAT_OUT2IN_NEXT_DROP;
2460 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2464 snat_det_reverse(dm0, &ip0->dst_address,
2465 clib_net_to_host_u16(tcp0->dst), &new_addr0);
2467 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2468 if (PREDICT_FALSE(!ses0))
2470 clib_warning("no match src %U:%d dst %U:%d for user %U",
2471 format_ip4_address, &ip0->src_address,
2472 clib_net_to_host_u16 (tcp0->src),
2473 format_ip4_address, &ip0->dst_address,
2474 clib_net_to_host_u16 (tcp0->dst),
2475 format_ip4_address, &new_addr0);
2476 next0 = SNAT_OUT2IN_NEXT_DROP;
2477 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2480 new_port0 = ses0->in_port;
2482 old_addr0 = ip0->dst_address;
2483 ip0->dst_address = new_addr0;
2484 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2486 sum0 = ip0->checksum;
2487 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2489 dst_address /* changed member */);
2490 ip0->checksum = ip_csum_fold (sum0);
2492 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2494 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2495 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2496 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2497 snat_det_ses_close(dm0, ses0);
2499 old_port0 = tcp0->dst;
2500 tcp0->dst = new_port0;
2502 sum0 = tcp0->checksum;
2503 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2505 dst_address /* changed member */);
2507 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2508 ip4_header_t /* cheat */,
2509 length /* changed member */);
2510 tcp0->checksum = ip_csum_fold(sum0);
2514 old_port0 = udp0->dst_port;
2515 udp0->dst_port = new_port0;
2521 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2522 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2524 snat_out2in_trace_t *t =
2525 vlib_add_trace (vm, node, b0, sizeof (*t));
2526 t->sw_if_index = sw_if_index0;
2527 t->next_index = next0;
2528 t->session_index = ~0;
2530 t->session_index = ses0 - dm0->sessions;
2533 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2535 /* verify speculative enqueue, maybe switch current next frame */
2536 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2537 to_next, n_left_to_next,
2541 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2544 vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2545 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2547 return frame->n_vectors;
2550 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2551 .function = snat_det_out2in_node_fn,
2552 .name = "nat44-det-out2in",
2553 .vector_size = sizeof (u32),
2554 .format_trace = format_snat_out2in_trace,
2555 .type = VLIB_NODE_TYPE_INTERNAL,
2557 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2558 .error_strings = snat_out2in_error_strings,
2560 .runtime_data_bytes = sizeof (snat_runtime_t),
2562 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2564 /* edit / add dispositions here */
2566 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2567 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2568 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2569 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2570 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2573 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2576 * Get address and port values to be used for ICMP packet translation
2577 * and create session if needed
2579 * @param[in,out] sm NAT main
2580 * @param[in,out] node NAT node runtime
2581 * @param[in] thread_index thread index
2582 * @param[in,out] b0 buffer containing packet to be translated
2583 * @param[out] p_proto protocol used for matching
2584 * @param[out] p_value address and port after NAT translation
2585 * @param[out] p_dont_translate if packet should not be translated
2586 * @param d optional parameter
2587 * @param e optional parameter
2589 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2590 u32 thread_index, vlib_buffer_t *b0,
2591 ip4_header_t *ip0, u8 *p_proto,
2592 snat_session_key_t *p_value,
2593 u8 *p_dont_translate, void *d, void *e)
2595 icmp46_header_t *icmp0;
2598 snat_det_out_key_t key0;
2599 u8 dont_translate = 0;
2601 icmp_echo_header_t *echo0, *inner_echo0 = 0;
2602 ip4_header_t *inner_ip0;
2603 void *l4_header = 0;
2604 icmp46_header_t *inner_icmp0;
2605 snat_det_map_t * dm0 = 0;
2606 ip4_address_t new_addr0 = {{0}};
2607 snat_det_session_t * ses0 = 0;
2608 ip4_address_t out_addr;
2610 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2611 echo0 = (icmp_echo_header_t *)(icmp0+1);
2612 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2614 if (!icmp_is_error_message (icmp0))
2616 protocol = SNAT_PROTOCOL_ICMP;
2617 key0.ext_host_addr = ip0->src_address;
2618 key0.ext_host_port = 0;
2619 key0.out_port = echo0->identifier;
2620 out_addr = ip0->dst_address;
2624 inner_ip0 = (ip4_header_t *)(echo0+1);
2625 l4_header = ip4_next_header (inner_ip0);
2626 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2627 key0.ext_host_addr = inner_ip0->dst_address;
2628 out_addr = inner_ip0->src_address;
2631 case SNAT_PROTOCOL_ICMP:
2632 inner_icmp0 = (icmp46_header_t*)l4_header;
2633 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2634 key0.ext_host_port = 0;
2635 key0.out_port = inner_echo0->identifier;
2637 case SNAT_PROTOCOL_UDP:
2638 case SNAT_PROTOCOL_TCP:
2639 key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2640 key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2643 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2644 next0 = SNAT_OUT2IN_NEXT_DROP;
2649 dm0 = snat_det_map_by_out(sm, &out_addr);
2650 if (PREDICT_FALSE(!dm0))
2652 /* Don't NAT packet aimed at the intfc address */
2653 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2654 ip0->dst_address.as_u32)))
2659 clib_warning("unknown dst address: %U",
2660 format_ip4_address, &ip0->dst_address);
2664 snat_det_reverse(dm0, &ip0->dst_address,
2665 clib_net_to_host_u16(key0.out_port), &new_addr0);
2667 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2668 if (PREDICT_FALSE(!ses0))
2670 /* Don't NAT packet aimed at the intfc address */
2671 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2672 ip0->dst_address.as_u32)))
2677 clib_warning("no match src %U:%d dst %U:%d for user %U",
2678 format_ip4_address, &key0.ext_host_addr,
2679 clib_net_to_host_u16 (key0.ext_host_port),
2680 format_ip4_address, &out_addr,
2681 clib_net_to_host_u16 (key0.out_port),
2682 format_ip4_address, &new_addr0);
2683 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2684 next0 = SNAT_OUT2IN_NEXT_DROP;
2688 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2689 !icmp_is_error_message (icmp0)))
2691 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2692 next0 = SNAT_OUT2IN_NEXT_DROP;
2699 *p_proto = protocol;
2702 p_value->addr = new_addr0;
2703 p_value->fib_index = sm->inside_fib_index;
2704 p_value->port = ses0->in_port;
2706 *p_dont_translate = dont_translate;
2708 *(snat_det_session_t**)d = ses0;
2710 *(snat_det_map_t**)e = dm0;
2714 /**********************/
2715 /*** worker handoff ***/
2716 /**********************/
2718 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2719 vlib_node_runtime_t * node,
2720 vlib_frame_t * frame)
2722 snat_main_t *sm = &snat_main;
2723 vlib_thread_main_t *tm = vlib_get_thread_main ();
2724 u32 n_left_from, *from, *to_next = 0;
2725 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2726 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2728 vlib_frame_queue_elt_t *hf = 0;
2729 vlib_frame_t *f = 0;
2731 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2732 u32 next_worker_index = 0;
2733 u32 current_worker_index = ~0;
2734 u32 thread_index = vlib_get_thread_index ();
2736 ASSERT (vec_len (sm->workers));
2738 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2740 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2742 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2743 sm->first_worker_index + sm->num_workers - 1,
2744 (vlib_frame_queue_t *) (~0));
2747 from = vlib_frame_vector_args (frame);
2748 n_left_from = frame->n_vectors;
2750 while (n_left_from > 0)
2763 b0 = vlib_get_buffer (vm, bi0);
2765 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2766 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2768 ip0 = vlib_buffer_get_current (b0);
2770 next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2772 if (PREDICT_FALSE (next_worker_index != thread_index))
2776 if (next_worker_index != current_worker_index)
2779 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2781 hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2783 handoff_queue_elt_by_worker_index);
2785 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2786 to_next_worker = &hf->buffer_index[hf->n_vectors];
2787 current_worker_index = next_worker_index;
2790 /* enqueue to correct worker thread */
2791 to_next_worker[0] = bi0;
2793 n_left_to_next_worker--;
2795 if (n_left_to_next_worker == 0)
2797 hf->n_vectors = VLIB_FRAME_SIZE;
2798 vlib_put_frame_queue_elt (hf);
2799 current_worker_index = ~0;
2800 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2807 /* if this is 1st frame */
2810 f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2811 to_next = vlib_frame_vector_args (f);
2819 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2820 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2822 snat_out2in_worker_handoff_trace_t *t =
2823 vlib_add_trace (vm, node, b0, sizeof (*t));
2824 t->next_worker_index = next_worker_index;
2825 t->do_handoff = do_handoff;
2830 vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2833 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2835 /* Ship frames to the worker nodes */
2836 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2838 if (handoff_queue_elt_by_worker_index[i])
2840 hf = handoff_queue_elt_by_worker_index[i];
2842 * It works better to let the handoff node
2843 * rate-adapt, always ship the handoff queue element.
2845 if (1 || hf->n_vectors == hf->last_n_vectors)
2847 vlib_put_frame_queue_elt (hf);
2848 handoff_queue_elt_by_worker_index[i] = 0;
2851 hf->last_n_vectors = hf->n_vectors;
2853 congested_handoff_queue_by_worker_index[i] =
2854 (vlib_frame_queue_t *) (~0);
2857 current_worker_index = ~0;
2858 return frame->n_vectors;
2861 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2862 .function = snat_out2in_worker_handoff_fn,
2863 .name = "nat44-out2in-worker-handoff",
2864 .vector_size = sizeof (u32),
2865 .format_trace = format_snat_out2in_worker_handoff_trace,
2866 .type = VLIB_NODE_TYPE_INTERNAL,
2875 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2878 snat_out2in_fast_node_fn (vlib_main_t * vm,
2879 vlib_node_runtime_t * node,
2880 vlib_frame_t * frame)
2882 u32 n_left_from, * from, * to_next;
2883 snat_out2in_next_t next_index;
2884 u32 pkts_processed = 0;
2885 snat_main_t * sm = &snat_main;
2887 from = vlib_frame_vector_args (frame);
2888 n_left_from = frame->n_vectors;
2889 next_index = node->cached_next_index;
2891 while (n_left_from > 0)
2895 vlib_get_next_frame (vm, node, next_index,
2896 to_next, n_left_to_next);
2898 while (n_left_from > 0 && n_left_to_next > 0)
2902 u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2906 u32 new_addr0, old_addr0;
2907 u16 new_port0, old_port0;
2908 udp_header_t * udp0;
2909 tcp_header_t * tcp0;
2910 icmp46_header_t * icmp0;
2911 snat_session_key_t key0, sm0;
2915 /* speculatively enqueue b0 to the current next frame */
2921 n_left_to_next -= 1;
2923 b0 = vlib_get_buffer (vm, bi0);
2925 ip0 = vlib_buffer_get_current (b0);
2926 udp0 = ip4_next_header (ip0);
2927 tcp0 = (tcp_header_t *) udp0;
2928 icmp0 = (icmp46_header_t *) udp0;
2930 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2931 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2933 vnet_feature_next (sw_if_index0, &next0, b0);
2935 if (PREDICT_FALSE(ip0->ttl == 1))
2937 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2938 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2939 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2941 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2945 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2947 if (PREDICT_FALSE (proto0 == ~0))
2950 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2952 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2953 rx_fib_index0, node, next0, ~0, 0, 0);
2957 key0.addr = ip0->dst_address;
2958 key0.port = udp0->dst_port;
2959 key0.fib_index = rx_fib_index0;
2961 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2963 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2967 new_addr0 = sm0.addr.as_u32;
2968 new_port0 = sm0.port;
2969 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2970 old_addr0 = ip0->dst_address.as_u32;
2971 ip0->dst_address.as_u32 = new_addr0;
2973 sum0 = ip0->checksum;
2974 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2976 dst_address /* changed member */);
2977 ip0->checksum = ip_csum_fold (sum0);
2979 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2981 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2983 old_port0 = tcp0->dst_port;
2984 tcp0->dst_port = new_port0;
2986 sum0 = tcp0->checksum;
2987 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2989 dst_address /* changed member */);
2991 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2992 ip4_header_t /* cheat */,
2993 length /* changed member */);
2994 tcp0->checksum = ip_csum_fold(sum0);
2998 old_port0 = udp0->dst_port;
2999 udp0->dst_port = new_port0;
3005 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3007 sum0 = tcp0->checksum;
3008 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3010 dst_address /* changed member */);
3012 tcp0->checksum = ip_csum_fold(sum0);
3018 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3019 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3021 snat_out2in_trace_t *t =
3022 vlib_add_trace (vm, node, b0, sizeof (*t));
3023 t->sw_if_index = sw_if_index0;
3024 t->next_index = next0;
3027 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3029 /* verify speculative enqueue, maybe switch current next frame */
3030 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3031 to_next, n_left_to_next,
3035 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3038 vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3039 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3041 return frame->n_vectors;
3044 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3045 .function = snat_out2in_fast_node_fn,
3046 .name = "nat44-out2in-fast",
3047 .vector_size = sizeof (u32),
3048 .format_trace = format_snat_out2in_fast_trace,
3049 .type = VLIB_NODE_TYPE_INTERNAL,
3051 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3052 .error_strings = snat_out2in_error_strings,
3054 .runtime_data_bytes = sizeof (snat_runtime_t),
3056 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3058 /* edit / add dispositions here */
3060 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3061 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3062 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3063 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3064 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
3067 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);