2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
26 #include <nat/nat_ipfix_logging.h>
27 #include <nat/nat_det.h>
28 #include <nat/nat_reass.h>
30 #include <vppinfra/hash.h>
31 #include <vppinfra/error.h>
32 #include <vppinfra/elog.h>
38 } snat_out2in_trace_t;
41 u32 next_worker_index;
43 } snat_out2in_worker_handoff_trace_t;
45 /* packet trace format function */
46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
48 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
49 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
50 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
52 s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
53 t->sw_if_index, t->next_index, t->session_index);
57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
59 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
60 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
61 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
63 s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
64 t->sw_if_index, t->next_index);
68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
70 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
71 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
72 snat_out2in_worker_handoff_trace_t * t =
73 va_arg (*args, snat_out2in_worker_handoff_trace_t *);
76 m = t->do_handoff ? "next worker" : "same worker";
77 s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
86 } nat44_out2in_reass_trace_t;
88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
90 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
91 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
92 nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
94 s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
95 t->sw_if_index, t->next_index,
96 t->cached ? "cached" : "translated");
101 vlib_node_registration_t snat_out2in_node;
102 vlib_node_registration_t snat_out2in_fast_node;
103 vlib_node_registration_t snat_out2in_worker_handoff_node;
104 vlib_node_registration_t snat_det_out2in_node;
105 vlib_node_registration_t nat44_out2in_reass_node;
107 #define foreach_snat_out2in_error \
108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
109 _(OUT2IN_PACKETS, "Good out2in packets processed") \
110 _(OUT_OF_PORTS, "Out of ports") \
111 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
112 _(NO_TRANSLATION, "No translation") \
113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
114 _(DROP_FRAGMENT, "Drop fragment") \
115 _(MAX_REASS, "Maximum reassemblies exceeded") \
116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
120 foreach_snat_out2in_error
123 } snat_out2in_error_t;
125 static char * snat_out2in_error_strings[] = {
126 #define _(sym,string) string,
127 foreach_snat_out2in_error
132 SNAT_OUT2IN_NEXT_DROP,
133 SNAT_OUT2IN_NEXT_LOOKUP,
134 SNAT_OUT2IN_NEXT_ICMP_ERROR,
135 SNAT_OUT2IN_NEXT_REASS,
136 SNAT_OUT2IN_NEXT_IN2OUT,
138 } snat_out2in_next_t;
141 * @brief Create session for static mapping.
143 * Create NAT session initiated by host from external network with static
146 * @param sm NAT main.
147 * @param b0 Vlib buffer.
148 * @param in2out In2out NAT44 session key.
149 * @param out2in Out2in NAT44 session key.
150 * @param node Vlib node.
152 * @returns SNAT session if successfully created otherwise 0.
154 static inline snat_session_t *
155 create_session_for_static_mapping (snat_main_t *sm,
157 snat_session_key_t in2out,
158 snat_session_key_t out2in,
159 vlib_node_runtime_t * node,
164 clib_bihash_kv_8_8_t kv0;
168 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
170 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
174 ip0 = vlib_buffer_get_current (b0);
175 udp0 = ip4_next_header (ip0);
177 u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
180 clib_warning ("create NAT user failed");
184 s = nat_session_alloc_or_recycle (sm, u, thread_index);
187 clib_warning ("create NAT session failed");
191 s->outside_address_index = ~0;
192 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
193 s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
194 s->ext_host_port = udp0->src_port;
195 user_session_increment (sm, u, 1 /* static */);
198 s->in2out.protocol = out2in.protocol;
200 /* Add to translation hashes */
201 kv0.key = s->in2out.as_u64;
202 kv0.value = s - sm->per_thread_data[thread_index].sessions;
203 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
205 clib_warning ("in2out key add failed");
207 kv0.key = s->out2in.as_u64;
209 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
211 clib_warning ("out2in key add failed");
214 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
215 s->out2in.addr.as_u32,
219 s->in2out.fib_index);
224 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
225 snat_session_key_t *p_key0)
227 icmp46_header_t *icmp0;
228 snat_session_key_t key0;
229 icmp_echo_header_t *echo0, *inner_echo0 = 0;
230 ip4_header_t *inner_ip0;
232 icmp46_header_t *inner_icmp0;
234 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
235 echo0 = (icmp_echo_header_t *)(icmp0+1);
237 if (!icmp_is_error_message (icmp0))
239 key0.protocol = SNAT_PROTOCOL_ICMP;
240 key0.addr = ip0->dst_address;
241 key0.port = echo0->identifier;
245 inner_ip0 = (ip4_header_t *)(echo0+1);
246 l4_header = ip4_next_header (inner_ip0);
247 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
248 key0.addr = inner_ip0->src_address;
249 switch (key0.protocol)
251 case SNAT_PROTOCOL_ICMP:
252 inner_icmp0 = (icmp46_header_t*)l4_header;
253 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
254 key0.port = inner_echo0->identifier;
256 case SNAT_PROTOCOL_UDP:
257 case SNAT_PROTOCOL_TCP:
258 key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
261 return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
265 return -1; /* success */
268 static_always_inline int
269 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
271 icmp46_header_t *icmp0;
272 nat_ed_ses_key_t key0;
273 icmp_echo_header_t *echo0, *inner_echo0 = 0;
274 ip4_header_t *inner_ip0;
276 icmp46_header_t *inner_icmp0;
278 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
279 echo0 = (icmp_echo_header_t *)(icmp0+1);
281 if (!icmp_is_error_message (icmp0))
283 key0.proto = IP_PROTOCOL_ICMP;
284 key0.l_addr = ip0->dst_address;
285 key0.r_addr = ip0->src_address;
286 key0.l_port = key0.r_port = echo0->identifier;
290 inner_ip0 = (ip4_header_t *)(echo0+1);
291 l4_header = ip4_next_header (inner_ip0);
292 key0.proto = inner_ip0->protocol;
293 key0.l_addr = inner_ip0->src_address;
294 key0.r_addr = inner_ip0->dst_address;
295 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
297 case SNAT_PROTOCOL_ICMP:
298 inner_icmp0 = (icmp46_header_t*)l4_header;
299 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
300 key0.l_port = key0.r_port = inner_echo0->identifier;
302 case SNAT_PROTOCOL_UDP:
303 case SNAT_PROTOCOL_TCP:
304 key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
305 key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
316 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u32 proto, u16 src_port,
319 snat_session_key_t key;
320 clib_bihash_kv_8_8_t kv, value;
322 key.addr = ip->src_address;
324 key.protocol = proto;
325 key.fib_index = sm->inside_fib_index;
328 if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv,
336 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
339 nat_ed_ses_key_t key;
340 clib_bihash_kv_16_8_t kv, value;
343 snat_session_t *s = 0;
344 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
346 if (ip->protocol == IP_PROTOCOL_ICMP)
348 if (icmp_get_ed_key (ip, &key))
351 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
353 udp = ip4_next_header(ip);
354 key.r_addr = ip->src_address;
355 key.l_addr = ip->dst_address;
356 key.proto = ip->protocol;
357 key.l_port = udp->dst_port;
358 key.r_port = udp->src_port;
362 key.r_addr = ip->src_address;
363 key.l_addr = ip->dst_address;
364 key.proto = ip->protocol;
365 key.l_port = key.r_port = 0;
368 kv.key[0] = key.as_u64[0];
369 kv.key[1] = key.as_u64[1];
371 if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
373 s = pool_elt_at_index (tsm->sessions, value.value);
377 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
380 u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index, thread_index);
383 clib_warning ("create NAT user failed");
387 s = nat_session_alloc_or_recycle (sm, u, thread_index);
390 clib_warning ("create NAT session failed");
394 s->ext_host_addr = key.r_addr;
395 s->ext_host_port = key.r_port;
396 s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
397 s->outside_address_index = ~0;
398 s->out2in.addr = key.l_addr;
399 s->out2in.port = key.l_port;
400 s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
401 s->out2in.fib_index = 0;
402 s->in2out = s->out2in;
403 user_session_increment (sm, u, 0);
405 kv.value = s - tsm->sessions;
406 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &kv, 1))
407 clib_warning ("in2out_ed key add failed");
410 /* Per-user LRU list maintenance */
411 clib_dlist_remove (tsm->list_pool, s->per_user_index);
412 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
417 * Get address and port values to be used for ICMP packet translation
418 * and create session if needed
420 * @param[in,out] sm NAT main
421 * @param[in,out] node NAT node runtime
422 * @param[in] thread_index thread index
423 * @param[in,out] b0 buffer containing packet to be translated
424 * @param[out] p_proto protocol used for matching
425 * @param[out] p_value address and port after NAT translation
426 * @param[out] p_dont_translate if packet should not be translated
427 * @param d optional parameter
428 * @param e optional parameter
430 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
431 u32 thread_index, vlib_buffer_t *b0,
432 ip4_header_t *ip0, u8 *p_proto,
433 snat_session_key_t *p_value,
434 u8 *p_dont_translate, void *d, void *e)
436 icmp46_header_t *icmp0;
439 snat_session_key_t key0;
440 snat_session_key_t sm0;
441 snat_session_t *s0 = 0;
442 u8 dont_translate = 0;
443 clib_bihash_kv_8_8_t kv0, value0;
448 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
449 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
450 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
454 err = icmp_get_key (ip0, &key0);
457 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
458 next0 = SNAT_OUT2IN_NEXT_DROP;
461 key0.fib_index = rx_fib_index0;
463 kv0.key = key0.as_u64;
465 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
468 /* Try to match static mapping by external address and port,
469 destination address and port in packet */
470 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
472 if (!sm->forwarding_enabled)
474 /* Don't NAT packet aimed at the intfc address */
475 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
476 ip0->dst_address.as_u32)))
481 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
482 next0 = SNAT_OUT2IN_NEXT_DROP;
488 if (next_src_nat(sm, ip0, key0.protocol, key0.port, thread_index))
490 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
493 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
498 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
499 (icmp0->type != ICMP4_echo_request || !is_addr_only)))
501 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
502 next0 = SNAT_OUT2IN_NEXT_DROP;
506 /* Create session initiated by host from external network */
507 s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
512 next0 = SNAT_OUT2IN_NEXT_DROP;
518 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
519 icmp0->type != ICMP4_echo_request &&
520 !icmp_is_error_message (icmp0)))
522 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
523 next0 = SNAT_OUT2IN_NEXT_DROP;
527 if (PREDICT_FALSE (value0.value == ~0ULL))
529 nat_ed_ses_key_t key;
530 clib_bihash_kv_16_8_t s_kv, s_value;
534 if (icmp_get_ed_key (ip0, &key))
536 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
537 next0 = SNAT_OUT2IN_NEXT_DROP;
540 key.fib_index = rx_fib_index0;
541 s_kv.key[0] = key.as_u64[0];
542 s_kv.key[1] = key.as_u64[1];
543 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
544 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
548 next0 = SNAT_OUT2IN_NEXT_DROP;
553 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
558 *p_proto = key0.protocol;
560 *p_value = s0->in2out;
561 *p_dont_translate = dont_translate;
563 *(snat_session_t**)d = s0;
568 * Get address and port values to be used for ICMP packet translation
570 * @param[in] sm NAT main
571 * @param[in,out] node NAT node runtime
572 * @param[in] thread_index thread index
573 * @param[in,out] b0 buffer containing packet to be translated
574 * @param[out] p_proto protocol used for matching
575 * @param[out] p_value address and port after NAT translation
576 * @param[out] p_dont_translate if packet should not be translated
577 * @param d optional parameter
578 * @param e optional parameter
580 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
581 u32 thread_index, vlib_buffer_t *b0,
582 ip4_header_t *ip0, u8 *p_proto,
583 snat_session_key_t *p_value,
584 u8 *p_dont_translate, void *d, void *e)
586 icmp46_header_t *icmp0;
589 snat_session_key_t key0;
590 snat_session_key_t sm0;
591 u8 dont_translate = 0;
596 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
597 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
598 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
600 err = icmp_get_key (ip0, &key0);
603 b0->error = node->errors[err];
604 next0 = SNAT_OUT2IN_NEXT_DROP;
607 key0.fib_index = rx_fib_index0;
609 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
611 /* Don't NAT packet aimed at the intfc address */
612 if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
617 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
618 next0 = SNAT_OUT2IN_NEXT_DROP;
622 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
623 (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
624 !icmp_is_error_message (icmp0)))
626 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
627 next0 = SNAT_OUT2IN_NEXT_DROP;
634 *p_proto = key0.protocol;
635 *p_dont_translate = dont_translate;
639 static inline u32 icmp_out2in (snat_main_t *sm,
642 icmp46_header_t * icmp0,
645 vlib_node_runtime_t * node,
651 snat_session_key_t sm0;
653 icmp_echo_header_t *echo0, *inner_echo0 = 0;
654 ip4_header_t *inner_ip0 = 0;
656 icmp46_header_t *inner_icmp0;
658 u32 new_addr0, old_addr0;
659 u16 old_id0, new_id0;
664 echo0 = (icmp_echo_header_t *)(icmp0+1);
666 next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
667 &protocol, &sm0, &dont_translate, d, e);
670 if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
673 sum0 = ip_incremental_checksum (0, icmp0,
674 ntohs(ip0->length) - ip4_header_bytes (ip0));
675 checksum0 = ~ip_csum_fold (sum0);
676 if (checksum0 != 0 && checksum0 != 0xffff)
678 next0 = SNAT_OUT2IN_NEXT_DROP;
682 old_addr0 = ip0->dst_address.as_u32;
683 new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
684 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
686 sum0 = ip0->checksum;
687 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
688 dst_address /* changed member */);
689 ip0->checksum = ip_csum_fold (sum0);
691 if (icmp0->checksum == 0)
692 icmp0->checksum = 0xffff;
694 if (!icmp_is_error_message (icmp0))
697 if (PREDICT_FALSE(new_id0 != echo0->identifier))
699 old_id0 = echo0->identifier;
701 echo0->identifier = new_id0;
703 sum0 = icmp0->checksum;
704 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
705 identifier /* changed member */);
706 icmp0->checksum = ip_csum_fold (sum0);
711 inner_ip0 = (ip4_header_t *)(echo0+1);
712 l4_header = ip4_next_header (inner_ip0);
714 if (!ip4_header_checksum_is_valid (inner_ip0))
716 next0 = SNAT_OUT2IN_NEXT_DROP;
720 old_addr0 = inner_ip0->src_address.as_u32;
721 inner_ip0->src_address = sm0.addr;
722 new_addr0 = inner_ip0->src_address.as_u32;
724 sum0 = icmp0->checksum;
725 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
726 src_address /* changed member */);
727 icmp0->checksum = ip_csum_fold (sum0);
731 case SNAT_PROTOCOL_ICMP:
732 inner_icmp0 = (icmp46_header_t*)l4_header;
733 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
735 old_id0 = inner_echo0->identifier;
737 inner_echo0->identifier = new_id0;
739 sum0 = icmp0->checksum;
740 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
742 icmp0->checksum = ip_csum_fold (sum0);
744 case SNAT_PROTOCOL_UDP:
745 case SNAT_PROTOCOL_TCP:
746 old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
748 ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
750 sum0 = icmp0->checksum;
751 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
753 icmp0->checksum = ip_csum_fold (sum0);
765 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
768 icmp46_header_t * icmp0,
771 vlib_node_runtime_t * node,
774 snat_session_t ** p_s0)
776 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
777 next0, thread_index, p_s0, 0);
778 snat_session_t * s0 = *p_s0;
779 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
782 s0->last_heard = now;
784 s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
785 /* Per-user LRU list maintenance */
786 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
788 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
789 s0->per_user_list_head_index,
795 static snat_session_t *
796 snat_out2in_unknown_proto (snat_main_t *sm,
803 vlib_node_runtime_t * node)
805 clib_bihash_kv_8_8_t kv, value;
806 clib_bihash_kv_16_8_t s_kv, s_value;
807 snat_static_mapping_t *m;
808 snat_session_key_t m_key;
809 u32 old_addr, new_addr;
811 nat_ed_ses_key_t key;
813 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
816 old_addr = ip->dst_address.as_u32;
818 key.l_addr = ip->dst_address;
819 key.r_addr = ip->src_address;
820 key.fib_index = rx_fib_index;
821 key.proto = ip->protocol;
824 s_kv.key[0] = key.as_u64[0];
825 s_kv.key[1] = key.as_u64[1];
827 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
829 s = pool_elt_at_index (tsm->sessions, s_value.value);
830 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
834 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
836 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
840 m_key.addr = ip->dst_address;
843 m_key.fib_index = rx_fib_index;
844 kv.key = m_key.as_u64;
845 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
847 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
851 m = pool_elt_at_index (sm->static_mappings, value.value);
853 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
855 u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
859 clib_warning ("create NAT user failed");
863 /* Create a new session */
864 s = nat_session_alloc_or_recycle (sm, u, thread_index);
867 clib_warning ("create NAT session failed");
871 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
872 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
873 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
874 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
875 s->outside_address_index = ~0;
876 s->out2in.addr.as_u32 = old_addr;
877 s->out2in.fib_index = rx_fib_index;
878 s->in2out.addr.as_u32 = new_addr;
879 s->in2out.fib_index = m->fib_index;
880 s->in2out.port = s->out2in.port = ip->protocol;
881 user_session_increment (sm, u, 1 /* static */);
883 /* Add to lookup tables */
884 s_kv.value = s - tsm->sessions;
885 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
886 clib_warning ("out2in key add failed");
888 key.l_addr = ip->dst_address;
889 key.fib_index = m->fib_index;
890 s_kv.key[0] = key.as_u64[0];
891 s_kv.key[1] = key.as_u64[1];
892 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
893 clib_warning ("in2out key add failed");
896 /* Update IP checksum */
898 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
899 ip->checksum = ip_csum_fold (sum);
901 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
906 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
907 /* Per-user LRU list maintenance */
908 clib_dlist_remove (tsm->list_pool, s->per_user_index);
909 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
915 static snat_session_t *
916 snat_out2in_lb (snat_main_t *sm,
923 vlib_node_runtime_t * node)
925 nat_ed_ses_key_t key;
926 clib_bihash_kv_16_8_t s_kv, s_value;
927 udp_header_t *udp = ip4_next_header (ip);
928 tcp_header_t *tcp = (tcp_header_t *) udp;
929 snat_session_t *s = 0;
930 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
931 snat_session_key_t e_key, l_key;
932 u32 old_addr, new_addr;
933 u32 proto = ip_proto_to_snat_proto (ip->protocol);
934 u16 new_port, old_port;
938 snat_session_key_t eh_key;
939 twice_nat_type_t twice_nat;
942 old_addr = ip->dst_address.as_u32;
944 key.l_addr = ip->dst_address;
945 key.r_addr = ip->src_address;
946 key.fib_index = rx_fib_index;
947 key.proto = ip->protocol;
948 key.r_port = udp->src_port;
949 key.l_port = udp->dst_port;
950 s_kv.key[0] = key.as_u64[0];
951 s_kv.key[1] = key.as_u64[1];
953 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
955 s = pool_elt_at_index (tsm->sessions, s_value.value);
959 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
961 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
965 e_key.addr = ip->dst_address;
966 e_key.port = udp->dst_port;
967 e_key.protocol = proto;
968 e_key.fib_index = rx_fib_index;
969 if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat, &lb))
972 u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
976 clib_warning ("create NAT user failed");
980 s = nat_session_alloc_or_recycle (sm, u, thread_index);
983 clib_warning ("create NAT session failed");
987 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
988 s->ext_host_port = udp->src_port;
989 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
991 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
992 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
993 s->outside_address_index = ~0;
996 user_session_increment (sm, u, 1 /* static */);
998 /* Add to lookup tables */
999 s_kv.value = s - tsm->sessions;
1000 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1001 clib_warning ("out2in-ed key add failed");
1003 if (twice_nat == TWICE_NAT ||
1004 (twice_nat == TWICE_NAT_SELF &&
1005 ip->src_address.as_u32 == l_key.addr.as_u32))
1007 eh_key.protocol = proto;
1008 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1009 thread_index, &eh_key,
1011 sm->port_per_thread,
1012 sm->per_thread_data[thread_index].snat_thread_index))
1014 b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1017 key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1018 key.r_port = s->ext_host_nat_port = eh_key.port;
1019 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1021 key.l_addr = l_key.addr;
1022 key.fib_index = l_key.fib_index;
1023 key.l_port = l_key.port;
1024 s_kv.key[0] = key.as_u64[0];
1025 s_kv.key[1] = key.as_u64[1];
1026 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1027 clib_warning ("in2out-ed key add failed");
1030 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1032 /* Update IP checksum */
1034 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1035 if (is_twice_nat_session (s))
1036 sum = ip_csum_update (sum, ip->src_address.as_u32,
1037 s->ext_host_nat_addr.as_u32, ip4_header_t,
1039 ip->checksum = ip_csum_fold (sum);
1041 if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1043 old_port = tcp->dst_port;
1044 tcp->dst_port = s->in2out.port;
1045 new_port = tcp->dst_port;
1047 sum = tcp->checksum;
1048 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1049 sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1050 if (is_twice_nat_session (s))
1052 sum = ip_csum_update (sum, ip->src_address.as_u32,
1053 s->ext_host_nat_addr.as_u32, ip4_header_t,
1055 sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
1056 ip4_header_t, length);
1057 tcp->src_port = s->ext_host_nat_port;
1058 ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1060 tcp->checksum = ip_csum_fold(sum);
1064 udp->dst_port = s->in2out.port;
1065 if (is_twice_nat_session (s))
1067 udp->src_port = s->ext_host_nat_port;
1068 ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
1073 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1076 s->last_heard = now;
1078 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1079 /* Per-user LRU list maintenance */
1080 clib_dlist_remove (tsm->list_pool, s->per_user_index);
1081 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1088 snat_out2in_node_fn (vlib_main_t * vm,
1089 vlib_node_runtime_t * node,
1090 vlib_frame_t * frame)
1092 u32 n_left_from, * from, * to_next;
1093 snat_out2in_next_t next_index;
1094 u32 pkts_processed = 0;
1095 snat_main_t * sm = &snat_main;
1096 f64 now = vlib_time_now (vm);
1097 u32 thread_index = vlib_get_thread_index ();
1099 from = vlib_frame_vector_args (frame);
1100 n_left_from = frame->n_vectors;
1101 next_index = node->cached_next_index;
1103 while (n_left_from > 0)
1107 vlib_get_next_frame (vm, node, next_index,
1108 to_next, n_left_to_next);
1110 while (n_left_from >= 4 && n_left_to_next >= 2)
1113 vlib_buffer_t * b0, * b1;
1114 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1115 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1116 u32 sw_if_index0, sw_if_index1;
1117 ip4_header_t * ip0, *ip1;
1118 ip_csum_t sum0, sum1;
1119 u32 new_addr0, old_addr0;
1120 u16 new_port0, old_port0;
1121 u32 new_addr1, old_addr1;
1122 u16 new_port1, old_port1;
1123 udp_header_t * udp0, * udp1;
1124 tcp_header_t * tcp0, * tcp1;
1125 icmp46_header_t * icmp0, * icmp1;
1126 snat_session_key_t key0, key1, sm0, sm1;
1127 u32 rx_fib_index0, rx_fib_index1;
1129 snat_session_t * s0 = 0, * s1 = 0;
1130 clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
1132 /* Prefetch next iteration. */
1134 vlib_buffer_t * p2, * p3;
1136 p2 = vlib_get_buffer (vm, from[2]);
1137 p3 = vlib_get_buffer (vm, from[3]);
1139 vlib_prefetch_buffer_header (p2, LOAD);
1140 vlib_prefetch_buffer_header (p3, LOAD);
1142 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1143 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1146 /* speculatively enqueue b0 and b1 to the current next frame */
1147 to_next[0] = bi0 = from[0];
1148 to_next[1] = bi1 = from[1];
1152 n_left_to_next -= 2;
1154 b0 = vlib_get_buffer (vm, bi0);
1155 b1 = vlib_get_buffer (vm, bi1);
1157 vnet_buffer (b0)->snat.flags = 0;
1158 vnet_buffer (b1)->snat.flags = 0;
1160 ip0 = vlib_buffer_get_current (b0);
1161 udp0 = ip4_next_header (ip0);
1162 tcp0 = (tcp_header_t *) udp0;
1163 icmp0 = (icmp46_header_t *) udp0;
1165 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1166 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1169 if (PREDICT_FALSE(ip0->ttl == 1))
1171 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1172 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1173 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1175 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1179 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1181 if (PREDICT_FALSE (proto0 == ~0))
1183 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1184 thread_index, now, vm, node);
1185 if (!sm->forwarding_enabled)
1187 next0 = SNAT_OUT2IN_NEXT_DROP;
1191 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1193 next0 = icmp_out2in_slow_path
1194 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1195 next0, now, thread_index, &s0);
1199 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1201 next0 = SNAT_OUT2IN_NEXT_REASS;
1205 key0.addr = ip0->dst_address;
1206 key0.port = udp0->dst_port;
1207 key0.protocol = proto0;
1208 key0.fib_index = rx_fib_index0;
1210 kv0.key = key0.as_u64;
1212 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1215 /* Try to match static mapping by external address and port,
1216 destination address and port in packet */
1217 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1220 * Send DHCP packets to the ipv4 stack, or we won't
1221 * be able to use dhcp client on the outside interface
1223 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1224 && (udp0->dst_port ==
1225 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1228 (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1232 if (!sm->forwarding_enabled)
1234 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1235 next0 = SNAT_OUT2IN_NEXT_DROP;
1240 if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1242 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1245 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1250 /* Create session initiated by host from external network */
1251 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1255 next0 = SNAT_OUT2IN_NEXT_DROP;
1261 if (PREDICT_FALSE (value0.value == ~0ULL))
1263 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1266 next0 = SNAT_OUT2IN_NEXT_DROP;
1271 s0 = pool_elt_at_index (
1272 sm->per_thread_data[thread_index].sessions,
1277 old_addr0 = ip0->dst_address.as_u32;
1278 ip0->dst_address = s0->in2out.addr;
1279 new_addr0 = ip0->dst_address.as_u32;
1280 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1282 sum0 = ip0->checksum;
1283 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1285 dst_address /* changed member */);
1286 ip0->checksum = ip_csum_fold (sum0);
1288 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1290 old_port0 = tcp0->dst_port;
1291 tcp0->dst_port = s0->in2out.port;
1292 new_port0 = tcp0->dst_port;
1294 sum0 = tcp0->checksum;
1295 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1297 dst_address /* changed member */);
1299 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1300 ip4_header_t /* cheat */,
1301 length /* changed member */);
1302 tcp0->checksum = ip_csum_fold(sum0);
1306 old_port0 = udp0->dst_port;
1307 udp0->dst_port = s0->in2out.port;
1312 s0->last_heard = now;
1314 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1315 /* Per-user LRU list maintenance */
1316 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1317 s0->per_user_index);
1318 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1319 s0->per_user_list_head_index,
1320 s0->per_user_index);
1323 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1324 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1326 snat_out2in_trace_t *t =
1327 vlib_add_trace (vm, node, b0, sizeof (*t));
1328 t->sw_if_index = sw_if_index0;
1329 t->next_index = next0;
1330 t->session_index = ~0;
1332 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1335 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1338 ip1 = vlib_buffer_get_current (b1);
1339 udp1 = ip4_next_header (ip1);
1340 tcp1 = (tcp_header_t *) udp1;
1341 icmp1 = (icmp46_header_t *) udp1;
1343 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1344 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1347 if (PREDICT_FALSE(ip1->ttl == 1))
1349 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1350 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1351 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1353 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1357 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1359 if (PREDICT_FALSE (proto1 == ~0))
1361 s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1362 thread_index, now, vm, node);
1363 if (!sm->forwarding_enabled)
1365 next1 = SNAT_OUT2IN_NEXT_DROP;
1369 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1371 next1 = icmp_out2in_slow_path
1372 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1373 next1, now, thread_index, &s1);
1377 if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1379 next1 = SNAT_OUT2IN_NEXT_REASS;
1383 key1.addr = ip1->dst_address;
1384 key1.port = udp1->dst_port;
1385 key1.protocol = proto1;
1386 key1.fib_index = rx_fib_index1;
1388 kv1.key = key1.as_u64;
1390 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1393 /* Try to match static mapping by external address and port,
1394 destination address and port in packet */
1395 if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
1398 * Send DHCP packets to the ipv4 stack, or we won't
1399 * be able to use dhcp client on the outside interface
1401 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
1402 && (udp1->dst_port ==
1403 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1406 (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
1410 if (!sm->forwarding_enabled)
1412 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1413 next1 = SNAT_OUT2IN_NEXT_DROP;
1418 if (next_src_nat(sm, ip1, proto1, udp1->src_port, thread_index))
1420 next1 = SNAT_OUT2IN_NEXT_IN2OUT;
1423 create_bypass_for_fwd(sm, ip1, rx_fib_index1, thread_index);
1428 /* Create session initiated by host from external network */
1429 s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1433 next1 = SNAT_OUT2IN_NEXT_DROP;
1439 if (PREDICT_FALSE (value1.value == ~0ULL))
1441 s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1444 next1 = SNAT_OUT2IN_NEXT_DROP;
1449 s1 = pool_elt_at_index (
1450 sm->per_thread_data[thread_index].sessions,
1455 old_addr1 = ip1->dst_address.as_u32;
1456 ip1->dst_address = s1->in2out.addr;
1457 new_addr1 = ip1->dst_address.as_u32;
1458 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1460 sum1 = ip1->checksum;
1461 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1463 dst_address /* changed member */);
1464 ip1->checksum = ip_csum_fold (sum1);
1466 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1468 old_port1 = tcp1->dst_port;
1469 tcp1->dst_port = s1->in2out.port;
1470 new_port1 = tcp1->dst_port;
1472 sum1 = tcp1->checksum;
1473 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1475 dst_address /* changed member */);
1477 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1478 ip4_header_t /* cheat */,
1479 length /* changed member */);
1480 tcp1->checksum = ip_csum_fold(sum1);
1484 old_port1 = udp1->dst_port;
1485 udp1->dst_port = s1->in2out.port;
1490 s1->last_heard = now;
1492 s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1493 /* Per-user LRU list maintenance */
1494 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1495 s1->per_user_index);
1496 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1497 s1->per_user_list_head_index,
1498 s1->per_user_index);
1501 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1502 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1504 snat_out2in_trace_t *t =
1505 vlib_add_trace (vm, node, b1, sizeof (*t));
1506 t->sw_if_index = sw_if_index1;
1507 t->next_index = next1;
1508 t->session_index = ~0;
1510 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1513 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1515 /* verify speculative enqueues, maybe switch current next frame */
1516 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1517 to_next, n_left_to_next,
1518 bi0, bi1, next0, next1);
1521 while (n_left_from > 0 && n_left_to_next > 0)
1525 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1529 u32 new_addr0, old_addr0;
1530 u16 new_port0, old_port0;
1531 udp_header_t * udp0;
1532 tcp_header_t * tcp0;
1533 icmp46_header_t * icmp0;
1534 snat_session_key_t key0, sm0;
1537 snat_session_t * s0 = 0;
1538 clib_bihash_kv_8_8_t kv0, value0;
1540 /* speculatively enqueue b0 to the current next frame */
1546 n_left_to_next -= 1;
1548 b0 = vlib_get_buffer (vm, bi0);
1550 vnet_buffer (b0)->snat.flags = 0;
1552 ip0 = vlib_buffer_get_current (b0);
1553 udp0 = ip4_next_header (ip0);
1554 tcp0 = (tcp_header_t *) udp0;
1555 icmp0 = (icmp46_header_t *) udp0;
1557 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1558 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1561 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1563 if (PREDICT_FALSE (proto0 == ~0))
1565 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1566 thread_index, now, vm, node);
1567 if (!sm->forwarding_enabled)
1569 next0 = SNAT_OUT2IN_NEXT_DROP;
1573 if (PREDICT_FALSE(ip0->ttl == 1))
1575 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1576 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1577 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1579 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1583 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1585 next0 = icmp_out2in_slow_path
1586 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1587 next0, now, thread_index, &s0);
1591 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1593 next0 = SNAT_OUT2IN_NEXT_REASS;
1597 key0.addr = ip0->dst_address;
1598 key0.port = udp0->dst_port;
1599 key0.protocol = proto0;
1600 key0.fib_index = rx_fib_index0;
1602 kv0.key = key0.as_u64;
1604 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1607 /* Try to match static mapping by external address and port,
1608 destination address and port in packet */
1609 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1612 * Send DHCP packets to the ipv4 stack, or we won't
1613 * be able to use dhcp client on the outside interface
1615 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1616 && (udp0->dst_port ==
1617 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1620 (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1624 if (!sm->forwarding_enabled)
1626 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1627 next0 = SNAT_OUT2IN_NEXT_DROP;
1632 if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1634 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1637 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1642 /* Create session initiated by host from external network */
1643 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1647 next0 = SNAT_OUT2IN_NEXT_DROP;
1653 if (PREDICT_FALSE (value0.value == ~0ULL))
1655 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1658 next0 = SNAT_OUT2IN_NEXT_DROP;
1663 s0 = pool_elt_at_index (
1664 sm->per_thread_data[thread_index].sessions,
1669 old_addr0 = ip0->dst_address.as_u32;
1670 ip0->dst_address = s0->in2out.addr;
1671 new_addr0 = ip0->dst_address.as_u32;
1672 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1674 sum0 = ip0->checksum;
1675 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1677 dst_address /* changed member */);
1678 ip0->checksum = ip_csum_fold (sum0);
1680 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1682 old_port0 = tcp0->dst_port;
1683 tcp0->dst_port = s0->in2out.port;
1684 new_port0 = tcp0->dst_port;
1686 sum0 = tcp0->checksum;
1687 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1689 dst_address /* changed member */);
1691 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1692 ip4_header_t /* cheat */,
1693 length /* changed member */);
1694 tcp0->checksum = ip_csum_fold(sum0);
1698 old_port0 = udp0->dst_port;
1699 udp0->dst_port = s0->in2out.port;
1704 s0->last_heard = now;
1706 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1707 /* Per-user LRU list maintenance */
1708 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1709 s0->per_user_index);
1710 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1711 s0->per_user_list_head_index,
1712 s0->per_user_index);
1715 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1716 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1718 snat_out2in_trace_t *t =
1719 vlib_add_trace (vm, node, b0, sizeof (*t));
1720 t->sw_if_index = sw_if_index0;
1721 t->next_index = next0;
1722 t->session_index = ~0;
1724 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1727 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1729 /* verify speculative enqueue, maybe switch current next frame */
1730 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1731 to_next, n_left_to_next,
1735 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1738 vlib_node_increment_counter (vm, snat_out2in_node.index,
1739 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1741 return frame->n_vectors;
1744 VLIB_REGISTER_NODE (snat_out2in_node) = {
1745 .function = snat_out2in_node_fn,
1746 .name = "nat44-out2in",
1747 .vector_size = sizeof (u32),
1748 .format_trace = format_snat_out2in_trace,
1749 .type = VLIB_NODE_TYPE_INTERNAL,
1751 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1752 .error_strings = snat_out2in_error_strings,
1754 .runtime_data_bytes = sizeof (snat_runtime_t),
1756 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1758 /* edit / add dispositions here */
1760 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1761 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1762 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1763 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1764 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
1767 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1770 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1771 vlib_node_runtime_t * node,
1772 vlib_frame_t * frame)
1774 u32 n_left_from, *from, *to_next;
1775 snat_out2in_next_t next_index;
1776 u32 pkts_processed = 0;
1777 snat_main_t *sm = &snat_main;
1778 f64 now = vlib_time_now (vm);
1779 u32 thread_index = vlib_get_thread_index ();
1780 snat_main_per_thread_data_t *per_thread_data =
1781 &sm->per_thread_data[thread_index];
1782 u32 *fragments_to_drop = 0;
1783 u32 *fragments_to_loopback = 0;
1785 from = vlib_frame_vector_args (frame);
1786 n_left_from = frame->n_vectors;
1787 next_index = node->cached_next_index;
1789 while (n_left_from > 0)
1793 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1795 while (n_left_from > 0 && n_left_to_next > 0)
1797 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1802 nat_reass_ip4_t *reass0;
1803 udp_header_t * udp0;
1804 tcp_header_t * tcp0;
1805 snat_session_key_t key0, sm0;
1806 clib_bihash_kv_8_8_t kv0, value0;
1807 snat_session_t * s0 = 0;
1808 u16 old_port0, new_port0;
1811 /* speculatively enqueue b0 to the current next frame */
1817 n_left_to_next -= 1;
1819 b0 = vlib_get_buffer (vm, bi0);
1820 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1822 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1823 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1826 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1828 next0 = SNAT_OUT2IN_NEXT_DROP;
1829 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1833 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1834 udp0 = ip4_next_header (ip0);
1835 tcp0 = (tcp_header_t *) udp0;
1836 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1838 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1843 &fragments_to_drop);
1845 if (PREDICT_FALSE (!reass0))
1847 next0 = SNAT_OUT2IN_NEXT_DROP;
1848 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1852 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1854 key0.addr = ip0->dst_address;
1855 key0.port = udp0->dst_port;
1856 key0.protocol = proto0;
1857 key0.fib_index = rx_fib_index0;
1858 kv0.key = key0.as_u64;
1860 if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1862 /* Try to match static mapping by external address and port,
1863 destination address and port in packet */
1864 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1867 * Send DHCP packets to the ipv4 stack, or we won't
1868 * be able to use dhcp client on the outside interface
1870 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1872 == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1875 (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1880 if (!sm->forwarding_enabled)
1882 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1883 next0 = SNAT_OUT2IN_NEXT_DROP;
1888 if (next_src_nat(sm, ip0, proto0, udp0->src_port, thread_index))
1890 next0 = SNAT_OUT2IN_NEXT_IN2OUT;
1893 create_bypass_for_fwd(sm, ip0, rx_fib_index0, thread_index);
1898 /* Create session initiated by host from external network */
1899 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1903 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1904 next0 = SNAT_OUT2IN_NEXT_DROP;
1907 reass0->sess_index = s0 - per_thread_data->sessions;
1908 reass0->thread_index = thread_index;
1912 s0 = pool_elt_at_index (per_thread_data->sessions,
1914 reass0->sess_index = value0.value;
1916 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1920 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1922 if (nat_ip4_reass_add_fragment (reass0, bi0))
1924 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1925 next0 = SNAT_OUT2IN_NEXT_DROP;
1931 s0 = pool_elt_at_index (per_thread_data->sessions,
1932 reass0->sess_index);
1935 old_addr0 = ip0->dst_address.as_u32;
1936 ip0->dst_address = s0->in2out.addr;
1937 new_addr0 = ip0->dst_address.as_u32;
1938 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1940 sum0 = ip0->checksum;
1941 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1943 dst_address /* changed member */);
1944 ip0->checksum = ip_csum_fold (sum0);
1946 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1948 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1950 old_port0 = tcp0->dst_port;
1951 tcp0->dst_port = s0->in2out.port;
1952 new_port0 = tcp0->dst_port;
1954 sum0 = tcp0->checksum;
1955 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1957 dst_address /* changed member */);
1959 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1960 ip4_header_t /* cheat */,
1961 length /* changed member */);
1962 tcp0->checksum = ip_csum_fold(sum0);
1966 old_port0 = udp0->dst_port;
1967 udp0->dst_port = s0->in2out.port;
1973 s0->last_heard = now;
1975 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1976 /* Per-user LRU list maintenance */
1977 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1978 s0->per_user_index);
1979 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1980 s0->per_user_list_head_index,
1981 s0->per_user_index);
1984 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1985 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1987 nat44_out2in_reass_trace_t *t =
1988 vlib_add_trace (vm, node, b0, sizeof (*t));
1989 t->cached = cached0;
1990 t->sw_if_index = sw_if_index0;
1991 t->next_index = next0;
2001 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2003 /* verify speculative enqueue, maybe switch current next frame */
2004 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2005 to_next, n_left_to_next,
2009 if (n_left_from == 0 && vec_len (fragments_to_loopback))
2011 from = vlib_frame_vector_args (frame);
2012 u32 len = vec_len (fragments_to_loopback);
2013 if (len <= VLIB_FRAME_SIZE)
2015 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2017 vec_reset_length (fragments_to_loopback);
2022 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2023 sizeof (u32) * VLIB_FRAME_SIZE);
2024 n_left_from = VLIB_FRAME_SIZE;
2025 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2030 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2033 vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
2034 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2037 nat_send_all_to_node (vm, fragments_to_drop, node,
2038 &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
2039 SNAT_OUT2IN_NEXT_DROP);
2041 vec_free (fragments_to_drop);
2042 vec_free (fragments_to_loopback);
2043 return frame->n_vectors;
2046 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
2047 .function = nat44_out2in_reass_node_fn,
2048 .name = "nat44-out2in-reass",
2049 .vector_size = sizeof (u32),
2050 .format_trace = format_nat44_out2in_reass_trace,
2051 .type = VLIB_NODE_TYPE_INTERNAL,
2053 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2054 .error_strings = snat_out2in_error_strings,
2056 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2058 /* edit / add dispositions here */
2060 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2061 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2062 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2063 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2064 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2067 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
2068 nat44_out2in_reass_node_fn);
2070 /**************************/
2071 /*** deterministic mode ***/
2072 /**************************/
2074 snat_det_out2in_node_fn (vlib_main_t * vm,
2075 vlib_node_runtime_t * node,
2076 vlib_frame_t * frame)
2078 u32 n_left_from, * from, * to_next;
2079 snat_out2in_next_t next_index;
2080 u32 pkts_processed = 0;
2081 snat_main_t * sm = &snat_main;
2082 u32 thread_index = vlib_get_thread_index ();
2084 from = vlib_frame_vector_args (frame);
2085 n_left_from = frame->n_vectors;
2086 next_index = node->cached_next_index;
2088 while (n_left_from > 0)
2092 vlib_get_next_frame (vm, node, next_index,
2093 to_next, n_left_to_next);
2095 while (n_left_from >= 4 && n_left_to_next >= 2)
2098 vlib_buffer_t * b0, * b1;
2099 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2100 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2101 u32 sw_if_index0, sw_if_index1;
2102 ip4_header_t * ip0, * ip1;
2103 ip_csum_t sum0, sum1;
2104 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2105 u16 new_port0, old_port0, old_port1, new_port1;
2106 udp_header_t * udp0, * udp1;
2107 tcp_header_t * tcp0, * tcp1;
2109 snat_det_out_key_t key0, key1;
2110 snat_det_map_t * dm0, * dm1;
2111 snat_det_session_t * ses0 = 0, * ses1 = 0;
2112 u32 rx_fib_index0, rx_fib_index1;
2113 icmp46_header_t * icmp0, * icmp1;
2115 /* Prefetch next iteration. */
2117 vlib_buffer_t * p2, * p3;
2119 p2 = vlib_get_buffer (vm, from[2]);
2120 p3 = vlib_get_buffer (vm, from[3]);
2122 vlib_prefetch_buffer_header (p2, LOAD);
2123 vlib_prefetch_buffer_header (p3, LOAD);
2125 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2126 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2129 /* speculatively enqueue b0 and b1 to the current next frame */
2130 to_next[0] = bi0 = from[0];
2131 to_next[1] = bi1 = from[1];
2135 n_left_to_next -= 2;
2137 b0 = vlib_get_buffer (vm, bi0);
2138 b1 = vlib_get_buffer (vm, bi1);
2140 ip0 = vlib_buffer_get_current (b0);
2141 udp0 = ip4_next_header (ip0);
2142 tcp0 = (tcp_header_t *) udp0;
2144 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2146 if (PREDICT_FALSE(ip0->ttl == 1))
2148 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2149 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2150 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2152 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2156 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2158 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2160 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2161 icmp0 = (icmp46_header_t *) udp0;
2163 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2164 rx_fib_index0, node, next0, thread_index,
2169 key0.ext_host_addr = ip0->src_address;
2170 key0.ext_host_port = tcp0->src;
2171 key0.out_port = tcp0->dst;
2173 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2174 if (PREDICT_FALSE(!dm0))
2176 clib_warning("unknown dst address: %U",
2177 format_ip4_address, &ip0->dst_address);
2178 next0 = SNAT_OUT2IN_NEXT_DROP;
2179 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2183 snat_det_reverse(dm0, &ip0->dst_address,
2184 clib_net_to_host_u16(tcp0->dst), &new_addr0);
2186 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2187 if (PREDICT_FALSE(!ses0))
2189 clib_warning("no match src %U:%d dst %U:%d for user %U",
2190 format_ip4_address, &ip0->src_address,
2191 clib_net_to_host_u16 (tcp0->src),
2192 format_ip4_address, &ip0->dst_address,
2193 clib_net_to_host_u16 (tcp0->dst),
2194 format_ip4_address, &new_addr0);
2195 next0 = SNAT_OUT2IN_NEXT_DROP;
2196 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2199 new_port0 = ses0->in_port;
2201 old_addr0 = ip0->dst_address;
2202 ip0->dst_address = new_addr0;
2203 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2205 sum0 = ip0->checksum;
2206 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2208 dst_address /* changed member */);
2209 ip0->checksum = ip_csum_fold (sum0);
2211 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2213 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2214 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2215 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2216 snat_det_ses_close(dm0, ses0);
2218 old_port0 = tcp0->dst;
2219 tcp0->dst = new_port0;
2221 sum0 = tcp0->checksum;
2222 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2224 dst_address /* changed member */);
2226 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2227 ip4_header_t /* cheat */,
2228 length /* changed member */);
2229 tcp0->checksum = ip_csum_fold(sum0);
2233 old_port0 = udp0->dst_port;
2234 udp0->dst_port = new_port0;
2240 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2241 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2243 snat_out2in_trace_t *t =
2244 vlib_add_trace (vm, node, b0, sizeof (*t));
2245 t->sw_if_index = sw_if_index0;
2246 t->next_index = next0;
2247 t->session_index = ~0;
2249 t->session_index = ses0 - dm0->sessions;
2252 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2254 b1 = vlib_get_buffer (vm, bi1);
2256 ip1 = vlib_buffer_get_current (b1);
2257 udp1 = ip4_next_header (ip1);
2258 tcp1 = (tcp_header_t *) udp1;
2260 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2262 if (PREDICT_FALSE(ip1->ttl == 1))
2264 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2265 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2266 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2268 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2272 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2274 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2276 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2277 icmp1 = (icmp46_header_t *) udp1;
2279 next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2280 rx_fib_index1, node, next1, thread_index,
2285 key1.ext_host_addr = ip1->src_address;
2286 key1.ext_host_port = tcp1->src;
2287 key1.out_port = tcp1->dst;
2289 dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2290 if (PREDICT_FALSE(!dm1))
2292 clib_warning("unknown dst address: %U",
2293 format_ip4_address, &ip1->dst_address);
2294 next1 = SNAT_OUT2IN_NEXT_DROP;
2295 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2299 snat_det_reverse(dm1, &ip1->dst_address,
2300 clib_net_to_host_u16(tcp1->dst), &new_addr1);
2302 ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2303 if (PREDICT_FALSE(!ses1))
2305 clib_warning("no match src %U:%d dst %U:%d for user %U",
2306 format_ip4_address, &ip1->src_address,
2307 clib_net_to_host_u16 (tcp1->src),
2308 format_ip4_address, &ip1->dst_address,
2309 clib_net_to_host_u16 (tcp1->dst),
2310 format_ip4_address, &new_addr1);
2311 next1 = SNAT_OUT2IN_NEXT_DROP;
2312 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2315 new_port1 = ses1->in_port;
2317 old_addr1 = ip1->dst_address;
2318 ip1->dst_address = new_addr1;
2319 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2321 sum1 = ip1->checksum;
2322 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2324 dst_address /* changed member */);
2325 ip1->checksum = ip_csum_fold (sum1);
2327 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2329 if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2330 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2331 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2332 snat_det_ses_close(dm1, ses1);
2334 old_port1 = tcp1->dst;
2335 tcp1->dst = new_port1;
2337 sum1 = tcp1->checksum;
2338 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2340 dst_address /* changed member */);
2342 sum1 = ip_csum_update (sum1, old_port1, new_port1,
2343 ip4_header_t /* cheat */,
2344 length /* changed member */);
2345 tcp1->checksum = ip_csum_fold(sum1);
2349 old_port1 = udp1->dst_port;
2350 udp1->dst_port = new_port1;
2356 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2357 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2359 snat_out2in_trace_t *t =
2360 vlib_add_trace (vm, node, b1, sizeof (*t));
2361 t->sw_if_index = sw_if_index1;
2362 t->next_index = next1;
2363 t->session_index = ~0;
2365 t->session_index = ses1 - dm1->sessions;
2368 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2370 /* verify speculative enqueues, maybe switch current next frame */
2371 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2372 to_next, n_left_to_next,
2373 bi0, bi1, next0, next1);
2376 while (n_left_from > 0 && n_left_to_next > 0)
2380 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2384 ip4_address_t new_addr0, old_addr0;
2385 u16 new_port0, old_port0;
2386 udp_header_t * udp0;
2387 tcp_header_t * tcp0;
2389 snat_det_out_key_t key0;
2390 snat_det_map_t * dm0;
2391 snat_det_session_t * ses0 = 0;
2393 icmp46_header_t * icmp0;
2395 /* speculatively enqueue b0 to the current next frame */
2401 n_left_to_next -= 1;
2403 b0 = vlib_get_buffer (vm, bi0);
2405 ip0 = vlib_buffer_get_current (b0);
2406 udp0 = ip4_next_header (ip0);
2407 tcp0 = (tcp_header_t *) udp0;
2409 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2411 if (PREDICT_FALSE(ip0->ttl == 1))
2413 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2414 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2415 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2417 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2421 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2423 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2425 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2426 icmp0 = (icmp46_header_t *) udp0;
2428 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2429 rx_fib_index0, node, next0, thread_index,
2434 key0.ext_host_addr = ip0->src_address;
2435 key0.ext_host_port = tcp0->src;
2436 key0.out_port = tcp0->dst;
2438 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2439 if (PREDICT_FALSE(!dm0))
2441 clib_warning("unknown dst address: %U",
2442 format_ip4_address, &ip0->dst_address);
2443 next0 = SNAT_OUT2IN_NEXT_DROP;
2444 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2448 snat_det_reverse(dm0, &ip0->dst_address,
2449 clib_net_to_host_u16(tcp0->dst), &new_addr0);
2451 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2452 if (PREDICT_FALSE(!ses0))
2454 clib_warning("no match src %U:%d dst %U:%d for user %U",
2455 format_ip4_address, &ip0->src_address,
2456 clib_net_to_host_u16 (tcp0->src),
2457 format_ip4_address, &ip0->dst_address,
2458 clib_net_to_host_u16 (tcp0->dst),
2459 format_ip4_address, &new_addr0);
2460 next0 = SNAT_OUT2IN_NEXT_DROP;
2461 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2464 new_port0 = ses0->in_port;
2466 old_addr0 = ip0->dst_address;
2467 ip0->dst_address = new_addr0;
2468 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2470 sum0 = ip0->checksum;
2471 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2473 dst_address /* changed member */);
2474 ip0->checksum = ip_csum_fold (sum0);
2476 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2478 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2479 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2480 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2481 snat_det_ses_close(dm0, ses0);
2483 old_port0 = tcp0->dst;
2484 tcp0->dst = new_port0;
2486 sum0 = tcp0->checksum;
2487 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2489 dst_address /* changed member */);
2491 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2492 ip4_header_t /* cheat */,
2493 length /* changed member */);
2494 tcp0->checksum = ip_csum_fold(sum0);
2498 old_port0 = udp0->dst_port;
2499 udp0->dst_port = new_port0;
2505 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2506 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2508 snat_out2in_trace_t *t =
2509 vlib_add_trace (vm, node, b0, sizeof (*t));
2510 t->sw_if_index = sw_if_index0;
2511 t->next_index = next0;
2512 t->session_index = ~0;
2514 t->session_index = ses0 - dm0->sessions;
2517 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2519 /* verify speculative enqueue, maybe switch current next frame */
2520 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2521 to_next, n_left_to_next,
2525 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2528 vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2529 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2531 return frame->n_vectors;
2534 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2535 .function = snat_det_out2in_node_fn,
2536 .name = "nat44-det-out2in",
2537 .vector_size = sizeof (u32),
2538 .format_trace = format_snat_out2in_trace,
2539 .type = VLIB_NODE_TYPE_INTERNAL,
2541 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2542 .error_strings = snat_out2in_error_strings,
2544 .runtime_data_bytes = sizeof (snat_runtime_t),
2546 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2548 /* edit / add dispositions here */
2550 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2551 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2552 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2553 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2554 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
2557 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2560 * Get address and port values to be used for ICMP packet translation
2561 * and create session if needed
2563 * @param[in,out] sm NAT main
2564 * @param[in,out] node NAT node runtime
2565 * @param[in] thread_index thread index
2566 * @param[in,out] b0 buffer containing packet to be translated
2567 * @param[out] p_proto protocol used for matching
2568 * @param[out] p_value address and port after NAT translation
2569 * @param[out] p_dont_translate if packet should not be translated
2570 * @param d optional parameter
2571 * @param e optional parameter
2573 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2574 u32 thread_index, vlib_buffer_t *b0,
2575 ip4_header_t *ip0, u8 *p_proto,
2576 snat_session_key_t *p_value,
2577 u8 *p_dont_translate, void *d, void *e)
2579 icmp46_header_t *icmp0;
2582 snat_det_out_key_t key0;
2583 u8 dont_translate = 0;
2585 icmp_echo_header_t *echo0, *inner_echo0 = 0;
2586 ip4_header_t *inner_ip0;
2587 void *l4_header = 0;
2588 icmp46_header_t *inner_icmp0;
2589 snat_det_map_t * dm0 = 0;
2590 ip4_address_t new_addr0 = {{0}};
2591 snat_det_session_t * ses0 = 0;
2592 ip4_address_t out_addr;
2594 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2595 echo0 = (icmp_echo_header_t *)(icmp0+1);
2596 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2598 if (!icmp_is_error_message (icmp0))
2600 protocol = SNAT_PROTOCOL_ICMP;
2601 key0.ext_host_addr = ip0->src_address;
2602 key0.ext_host_port = 0;
2603 key0.out_port = echo0->identifier;
2604 out_addr = ip0->dst_address;
2608 inner_ip0 = (ip4_header_t *)(echo0+1);
2609 l4_header = ip4_next_header (inner_ip0);
2610 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2611 key0.ext_host_addr = inner_ip0->dst_address;
2612 out_addr = inner_ip0->src_address;
2615 case SNAT_PROTOCOL_ICMP:
2616 inner_icmp0 = (icmp46_header_t*)l4_header;
2617 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2618 key0.ext_host_port = 0;
2619 key0.out_port = inner_echo0->identifier;
2621 case SNAT_PROTOCOL_UDP:
2622 case SNAT_PROTOCOL_TCP:
2623 key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2624 key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2627 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2628 next0 = SNAT_OUT2IN_NEXT_DROP;
2633 dm0 = snat_det_map_by_out(sm, &out_addr);
2634 if (PREDICT_FALSE(!dm0))
2636 /* Don't NAT packet aimed at the intfc address */
2637 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2638 ip0->dst_address.as_u32)))
2643 clib_warning("unknown dst address: %U",
2644 format_ip4_address, &ip0->dst_address);
2648 snat_det_reverse(dm0, &ip0->dst_address,
2649 clib_net_to_host_u16(key0.out_port), &new_addr0);
2651 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2652 if (PREDICT_FALSE(!ses0))
2654 /* Don't NAT packet aimed at the intfc address */
2655 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2656 ip0->dst_address.as_u32)))
2661 clib_warning("no match src %U:%d dst %U:%d for user %U",
2662 format_ip4_address, &key0.ext_host_addr,
2663 clib_net_to_host_u16 (key0.ext_host_port),
2664 format_ip4_address, &out_addr,
2665 clib_net_to_host_u16 (key0.out_port),
2666 format_ip4_address, &new_addr0);
2667 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2668 next0 = SNAT_OUT2IN_NEXT_DROP;
2672 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2673 !icmp_is_error_message (icmp0)))
2675 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2676 next0 = SNAT_OUT2IN_NEXT_DROP;
2683 *p_proto = protocol;
2686 p_value->addr = new_addr0;
2687 p_value->fib_index = sm->inside_fib_index;
2688 p_value->port = ses0->in_port;
2690 *p_dont_translate = dont_translate;
2692 *(snat_det_session_t**)d = ses0;
2694 *(snat_det_map_t**)e = dm0;
2698 /**********************/
2699 /*** worker handoff ***/
2700 /**********************/
2702 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2703 vlib_node_runtime_t * node,
2704 vlib_frame_t * frame)
2706 snat_main_t *sm = &snat_main;
2707 vlib_thread_main_t *tm = vlib_get_thread_main ();
2708 u32 n_left_from, *from, *to_next = 0;
2709 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2710 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2712 vlib_frame_queue_elt_t *hf = 0;
2713 vlib_frame_t *f = 0;
2715 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2716 u32 next_worker_index = 0;
2717 u32 current_worker_index = ~0;
2718 u32 thread_index = vlib_get_thread_index ();
2720 ASSERT (vec_len (sm->workers));
2722 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2724 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2726 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2727 sm->first_worker_index + sm->num_workers - 1,
2728 (vlib_frame_queue_t *) (~0));
2731 from = vlib_frame_vector_args (frame);
2732 n_left_from = frame->n_vectors;
2734 while (n_left_from > 0)
2747 b0 = vlib_get_buffer (vm, bi0);
2749 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2750 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2752 ip0 = vlib_buffer_get_current (b0);
2754 next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2756 if (PREDICT_FALSE (next_worker_index != thread_index))
2760 if (next_worker_index != current_worker_index)
2763 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2765 hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2767 handoff_queue_elt_by_worker_index);
2769 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2770 to_next_worker = &hf->buffer_index[hf->n_vectors];
2771 current_worker_index = next_worker_index;
2774 /* enqueue to correct worker thread */
2775 to_next_worker[0] = bi0;
2777 n_left_to_next_worker--;
2779 if (n_left_to_next_worker == 0)
2781 hf->n_vectors = VLIB_FRAME_SIZE;
2782 vlib_put_frame_queue_elt (hf);
2783 current_worker_index = ~0;
2784 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2791 /* if this is 1st frame */
2794 f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2795 to_next = vlib_frame_vector_args (f);
2803 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2804 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2806 snat_out2in_worker_handoff_trace_t *t =
2807 vlib_add_trace (vm, node, b0, sizeof (*t));
2808 t->next_worker_index = next_worker_index;
2809 t->do_handoff = do_handoff;
2814 vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2817 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2819 /* Ship frames to the worker nodes */
2820 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2822 if (handoff_queue_elt_by_worker_index[i])
2824 hf = handoff_queue_elt_by_worker_index[i];
2826 * It works better to let the handoff node
2827 * rate-adapt, always ship the handoff queue element.
2829 if (1 || hf->n_vectors == hf->last_n_vectors)
2831 vlib_put_frame_queue_elt (hf);
2832 handoff_queue_elt_by_worker_index[i] = 0;
2835 hf->last_n_vectors = hf->n_vectors;
2837 congested_handoff_queue_by_worker_index[i] =
2838 (vlib_frame_queue_t *) (~0);
2841 current_worker_index = ~0;
2842 return frame->n_vectors;
2845 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2846 .function = snat_out2in_worker_handoff_fn,
2847 .name = "nat44-out2in-worker-handoff",
2848 .vector_size = sizeof (u32),
2849 .format_trace = format_snat_out2in_worker_handoff_trace,
2850 .type = VLIB_NODE_TYPE_INTERNAL,
2859 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2862 snat_out2in_fast_node_fn (vlib_main_t * vm,
2863 vlib_node_runtime_t * node,
2864 vlib_frame_t * frame)
2866 u32 n_left_from, * from, * to_next;
2867 snat_out2in_next_t next_index;
2868 u32 pkts_processed = 0;
2869 snat_main_t * sm = &snat_main;
2871 from = vlib_frame_vector_args (frame);
2872 n_left_from = frame->n_vectors;
2873 next_index = node->cached_next_index;
2875 while (n_left_from > 0)
2879 vlib_get_next_frame (vm, node, next_index,
2880 to_next, n_left_to_next);
2882 while (n_left_from > 0 && n_left_to_next > 0)
2886 u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2890 u32 new_addr0, old_addr0;
2891 u16 new_port0, old_port0;
2892 udp_header_t * udp0;
2893 tcp_header_t * tcp0;
2894 icmp46_header_t * icmp0;
2895 snat_session_key_t key0, sm0;
2899 /* speculatively enqueue b0 to the current next frame */
2905 n_left_to_next -= 1;
2907 b0 = vlib_get_buffer (vm, bi0);
2909 ip0 = vlib_buffer_get_current (b0);
2910 udp0 = ip4_next_header (ip0);
2911 tcp0 = (tcp_header_t *) udp0;
2912 icmp0 = (icmp46_header_t *) udp0;
2914 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2915 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2917 vnet_feature_next (sw_if_index0, &next0, b0);
2919 if (PREDICT_FALSE(ip0->ttl == 1))
2921 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2922 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2923 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2925 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2929 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2931 if (PREDICT_FALSE (proto0 == ~0))
2934 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2936 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2937 rx_fib_index0, node, next0, ~0, 0, 0);
2941 key0.addr = ip0->dst_address;
2942 key0.port = udp0->dst_port;
2943 key0.fib_index = rx_fib_index0;
2945 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2947 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2951 new_addr0 = sm0.addr.as_u32;
2952 new_port0 = sm0.port;
2953 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2954 old_addr0 = ip0->dst_address.as_u32;
2955 ip0->dst_address.as_u32 = new_addr0;
2957 sum0 = ip0->checksum;
2958 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2960 dst_address /* changed member */);
2961 ip0->checksum = ip_csum_fold (sum0);
2963 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2965 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2967 old_port0 = tcp0->dst_port;
2968 tcp0->dst_port = new_port0;
2970 sum0 = tcp0->checksum;
2971 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2973 dst_address /* changed member */);
2975 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2976 ip4_header_t /* cheat */,
2977 length /* changed member */);
2978 tcp0->checksum = ip_csum_fold(sum0);
2982 old_port0 = udp0->dst_port;
2983 udp0->dst_port = new_port0;
2989 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2991 sum0 = tcp0->checksum;
2992 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2994 dst_address /* changed member */);
2996 tcp0->checksum = ip_csum_fold(sum0);
3002 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3003 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3005 snat_out2in_trace_t *t =
3006 vlib_add_trace (vm, node, b0, sizeof (*t));
3007 t->sw_if_index = sw_if_index0;
3008 t->next_index = next0;
3011 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3013 /* verify speculative enqueue, maybe switch current next frame */
3014 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3015 to_next, n_left_to_next,
3019 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3022 vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3023 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3025 return frame->n_vectors;
3028 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3029 .function = snat_out2in_fast_node_fn,
3030 .name = "nat44-out2in-fast",
3031 .vector_size = sizeof (u32),
3032 .format_trace = format_snat_out2in_fast_trace,
3033 .type = VLIB_NODE_TYPE_INTERNAL,
3035 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3036 .error_strings = snat_out2in_error_strings,
3038 .runtime_data_bytes = sizeof (snat_runtime_t),
3040 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3042 /* edit / add dispositions here */
3044 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3045 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3046 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3047 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3048 [SNAT_OUT2IN_NEXT_IN2OUT] = "nat44-in2out",
3051 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);