2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
26 #include <nat/nat_ipfix_logging.h>
27 #include <nat/nat_det.h>
28 #include <nat/nat_reass.h>
30 #include <vppinfra/hash.h>
31 #include <vppinfra/error.h>
32 #include <vppinfra/elog.h>
38 } snat_out2in_trace_t;
41 u32 next_worker_index;
43 } snat_out2in_worker_handoff_trace_t;
45 /* packet trace format function */
46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
48 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
49 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
50 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
52 s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
53 t->sw_if_index, t->next_index, t->session_index);
57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
59 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
60 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
61 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
63 s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
64 t->sw_if_index, t->next_index);
68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
70 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
71 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
72 snat_out2in_worker_handoff_trace_t * t =
73 va_arg (*args, snat_out2in_worker_handoff_trace_t *);
76 m = t->do_handoff ? "next worker" : "same worker";
77 s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
86 } nat44_out2in_reass_trace_t;
88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
90 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
91 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
92 nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
94 s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
95 t->sw_if_index, t->next_index,
96 t->cached ? "cached" : "translated");
101 vlib_node_registration_t snat_out2in_node;
102 vlib_node_registration_t snat_out2in_fast_node;
103 vlib_node_registration_t snat_out2in_worker_handoff_node;
104 vlib_node_registration_t snat_det_out2in_node;
105 vlib_node_registration_t nat44_out2in_reass_node;
107 #define foreach_snat_out2in_error \
108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
109 _(OUT2IN_PACKETS, "Good out2in packets processed") \
110 _(OUT_OF_PORTS, "Out of ports") \
111 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
112 _(NO_TRANSLATION, "No translation") \
113 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
114 _(DROP_FRAGMENT, "Drop fragment") \
115 _(MAX_REASS, "Maximum reassemblies exceeded") \
116 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
119 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
120 foreach_snat_out2in_error
123 } snat_out2in_error_t;
125 static char * snat_out2in_error_strings[] = {
126 #define _(sym,string) string,
127 foreach_snat_out2in_error
132 SNAT_OUT2IN_NEXT_DROP,
133 SNAT_OUT2IN_NEXT_LOOKUP,
134 SNAT_OUT2IN_NEXT_ICMP_ERROR,
135 SNAT_OUT2IN_NEXT_REASS,
137 } snat_out2in_next_t;
140 * @brief Create session for static mapping.
142 * Create NAT session initiated by host from external network with static
145 * @param sm NAT main.
146 * @param b0 Vlib buffer.
147 * @param in2out In2out NAT44 session key.
148 * @param out2in Out2in NAT44 session key.
149 * @param node Vlib node.
151 * @returns SNAT session if successfully created otherwise 0.
153 static inline snat_session_t *
154 create_session_for_static_mapping (snat_main_t *sm,
156 snat_session_key_t in2out,
157 snat_session_key_t out2in,
158 vlib_node_runtime_t * node,
163 clib_bihash_kv_8_8_t kv0;
167 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
169 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
173 ip0 = vlib_buffer_get_current (b0);
174 udp0 = ip4_next_header (ip0);
176 u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
179 clib_warning ("create NAT user failed");
183 s = nat_session_alloc_or_recycle (sm, u, thread_index);
186 clib_warning ("create NAT session failed");
190 s->outside_address_index = ~0;
191 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
192 s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
193 s->ext_host_port = udp0->src_port;
194 u->nstaticsessions++;
197 s->in2out.protocol = out2in.protocol;
199 /* Add to translation hashes */
200 kv0.key = s->in2out.as_u64;
201 kv0.value = s - sm->per_thread_data[thread_index].sessions;
202 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
204 clib_warning ("in2out key add failed");
206 kv0.key = s->out2in.as_u64;
208 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
210 clib_warning ("out2in key add failed");
213 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
214 s->out2in.addr.as_u32,
218 s->in2out.fib_index);
223 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
224 snat_session_key_t *p_key0)
226 icmp46_header_t *icmp0;
227 snat_session_key_t key0;
228 icmp_echo_header_t *echo0, *inner_echo0 = 0;
229 ip4_header_t *inner_ip0;
231 icmp46_header_t *inner_icmp0;
233 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
234 echo0 = (icmp_echo_header_t *)(icmp0+1);
236 if (!icmp_is_error_message (icmp0))
238 key0.protocol = SNAT_PROTOCOL_ICMP;
239 key0.addr = ip0->dst_address;
240 key0.port = echo0->identifier;
244 inner_ip0 = (ip4_header_t *)(echo0+1);
245 l4_header = ip4_next_header (inner_ip0);
246 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
247 key0.addr = inner_ip0->src_address;
248 switch (key0.protocol)
250 case SNAT_PROTOCOL_ICMP:
251 inner_icmp0 = (icmp46_header_t*)l4_header;
252 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
253 key0.port = inner_echo0->identifier;
255 case SNAT_PROTOCOL_UDP:
256 case SNAT_PROTOCOL_TCP:
257 key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
260 return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
264 return -1; /* success */
268 * Get address and port values to be used for ICMP packet translation
269 * and create session if needed
271 * @param[in,out] sm NAT main
272 * @param[in,out] node NAT node runtime
273 * @param[in] thread_index thread index
274 * @param[in,out] b0 buffer containing packet to be translated
275 * @param[out] p_proto protocol used for matching
276 * @param[out] p_value address and port after NAT translation
277 * @param[out] p_dont_translate if packet should not be translated
278 * @param d optional parameter
279 * @param e optional parameter
281 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
282 u32 thread_index, vlib_buffer_t *b0,
283 ip4_header_t *ip0, u8 *p_proto,
284 snat_session_key_t *p_value,
285 u8 *p_dont_translate, void *d, void *e)
287 icmp46_header_t *icmp0;
290 snat_session_key_t key0;
291 snat_session_key_t sm0;
292 snat_session_t *s0 = 0;
293 u8 dont_translate = 0;
294 clib_bihash_kv_8_8_t kv0, value0;
299 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
300 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
301 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
305 err = icmp_get_key (ip0, &key0);
308 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
309 next0 = SNAT_OUT2IN_NEXT_DROP;
312 key0.fib_index = rx_fib_index0;
314 kv0.key = key0.as_u64;
316 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
319 /* Try to match static mapping by external address and port,
320 destination address and port in packet */
321 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0))
323 if (!sm->forwarding_enabled)
325 /* Don't NAT packet aimed at the intfc address */
326 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
327 ip0->dst_address.as_u32)))
332 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
333 next0 = SNAT_OUT2IN_NEXT_DROP;
343 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
344 (icmp0->type != ICMP4_echo_request || !is_addr_only)))
346 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
347 next0 = SNAT_OUT2IN_NEXT_DROP;
351 /* Create session initiated by host from external network */
352 s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
357 next0 = SNAT_OUT2IN_NEXT_DROP;
363 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
364 icmp0->type != ICMP4_echo_request &&
365 !icmp_is_error_message (icmp0)))
367 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
368 next0 = SNAT_OUT2IN_NEXT_DROP;
372 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
377 *p_proto = key0.protocol;
379 *p_value = s0->in2out;
380 *p_dont_translate = dont_translate;
382 *(snat_session_t**)d = s0;
387 * Get address and port values to be used for ICMP packet translation
389 * @param[in] sm NAT main
390 * @param[in,out] node NAT node runtime
391 * @param[in] thread_index thread index
392 * @param[in,out] b0 buffer containing packet to be translated
393 * @param[out] p_proto protocol used for matching
394 * @param[out] p_value address and port after NAT translation
395 * @param[out] p_dont_translate if packet should not be translated
396 * @param d optional parameter
397 * @param e optional parameter
399 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
400 u32 thread_index, vlib_buffer_t *b0,
401 ip4_header_t *ip0, u8 *p_proto,
402 snat_session_key_t *p_value,
403 u8 *p_dont_translate, void *d, void *e)
405 icmp46_header_t *icmp0;
408 snat_session_key_t key0;
409 snat_session_key_t sm0;
410 u8 dont_translate = 0;
415 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
416 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
417 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
419 err = icmp_get_key (ip0, &key0);
422 b0->error = node->errors[err];
423 next0 = SNAT_OUT2IN_NEXT_DROP;
426 key0.fib_index = rx_fib_index0;
428 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0))
430 /* Don't NAT packet aimed at the intfc address */
431 if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
436 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
437 next0 = SNAT_OUT2IN_NEXT_DROP;
441 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
442 (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
443 !icmp_is_error_message (icmp0)))
445 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
446 next0 = SNAT_OUT2IN_NEXT_DROP;
453 *p_proto = key0.protocol;
454 *p_dont_translate = dont_translate;
458 static inline u32 icmp_out2in (snat_main_t *sm,
461 icmp46_header_t * icmp0,
464 vlib_node_runtime_t * node,
470 snat_session_key_t sm0;
472 icmp_echo_header_t *echo0, *inner_echo0 = 0;
473 ip4_header_t *inner_ip0 = 0;
475 icmp46_header_t *inner_icmp0;
477 u32 new_addr0, old_addr0;
478 u16 old_id0, new_id0;
483 echo0 = (icmp_echo_header_t *)(icmp0+1);
485 next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
486 &protocol, &sm0, &dont_translate, d, e);
489 if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
492 sum0 = ip_incremental_checksum (0, icmp0,
493 ntohs(ip0->length) - ip4_header_bytes (ip0));
494 checksum0 = ~ip_csum_fold (sum0);
495 if (checksum0 != 0 && checksum0 != 0xffff)
497 next0 = SNAT_OUT2IN_NEXT_DROP;
501 old_addr0 = ip0->dst_address.as_u32;
502 new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
503 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
505 sum0 = ip0->checksum;
506 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
507 dst_address /* changed member */);
508 ip0->checksum = ip_csum_fold (sum0);
510 if (!icmp_is_error_message (icmp0))
513 if (PREDICT_FALSE(new_id0 != echo0->identifier))
515 old_id0 = echo0->identifier;
517 echo0->identifier = new_id0;
519 sum0 = icmp0->checksum;
520 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
521 identifier /* changed member */);
522 icmp0->checksum = ip_csum_fold (sum0);
527 inner_ip0 = (ip4_header_t *)(echo0+1);
528 l4_header = ip4_next_header (inner_ip0);
530 if (!ip4_header_checksum_is_valid (inner_ip0))
532 next0 = SNAT_OUT2IN_NEXT_DROP;
536 old_addr0 = inner_ip0->src_address.as_u32;
537 inner_ip0->src_address = sm0.addr;
538 new_addr0 = inner_ip0->src_address.as_u32;
540 sum0 = icmp0->checksum;
541 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
542 src_address /* changed member */);
543 icmp0->checksum = ip_csum_fold (sum0);
547 case SNAT_PROTOCOL_ICMP:
548 inner_icmp0 = (icmp46_header_t*)l4_header;
549 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
551 old_id0 = inner_echo0->identifier;
553 inner_echo0->identifier = new_id0;
555 sum0 = icmp0->checksum;
556 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
558 icmp0->checksum = ip_csum_fold (sum0);
560 case SNAT_PROTOCOL_UDP:
561 case SNAT_PROTOCOL_TCP:
562 old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
564 ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
566 sum0 = icmp0->checksum;
567 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
569 icmp0->checksum = ip_csum_fold (sum0);
581 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
584 icmp46_header_t * icmp0,
587 vlib_node_runtime_t * node,
590 snat_session_t ** p_s0)
592 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
593 next0, thread_index, p_s0, 0);
594 snat_session_t * s0 = *p_s0;
595 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
598 s0->last_heard = now;
600 s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
601 /* Per-user LRU list maintenance */
602 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
604 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
605 s0->per_user_list_head_index,
611 static snat_session_t *
612 snat_out2in_unknown_proto (snat_main_t *sm,
619 vlib_node_runtime_t * node)
621 clib_bihash_kv_8_8_t kv, value;
622 clib_bihash_kv_16_8_t s_kv, s_value;
623 snat_static_mapping_t *m;
624 snat_session_key_t m_key;
625 u32 old_addr, new_addr;
627 nat_ed_ses_key_t key;
629 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
632 old_addr = ip->dst_address.as_u32;
634 key.l_addr = ip->dst_address;
635 key.r_addr = ip->src_address;
636 key.fib_index = rx_fib_index;
637 key.proto = ip->protocol;
640 s_kv.key[0] = key.as_u64[0];
641 s_kv.key[1] = key.as_u64[1];
643 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
645 s = pool_elt_at_index (tsm->sessions, s_value.value);
646 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
650 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
652 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
656 m_key.addr = ip->dst_address;
659 m_key.fib_index = rx_fib_index;
660 kv.key = m_key.as_u64;
661 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
663 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
667 m = pool_elt_at_index (sm->static_mappings, value.value);
669 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
671 u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
675 clib_warning ("create NAT user failed");
679 /* Create a new session */
680 s = nat_session_alloc_or_recycle (sm, u, thread_index);
683 clib_warning ("create NAT session failed");
687 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
688 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
689 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
690 s->outside_address_index = ~0;
691 s->out2in.addr.as_u32 = old_addr;
692 s->out2in.fib_index = rx_fib_index;
693 s->in2out.addr.as_u32 = new_addr;
694 s->in2out.fib_index = m->fib_index;
695 s->in2out.port = s->out2in.port = ip->protocol;
696 u->nstaticsessions++;
698 /* Add to lookup tables */
699 s_kv.value = s - tsm->sessions;
700 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
701 clib_warning ("out2in key add failed");
703 key.l_addr = ip->dst_address;
704 key.fib_index = m->fib_index;
705 s_kv.key[0] = key.as_u64[0];
706 s_kv.key[1] = key.as_u64[1];
707 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
708 clib_warning ("in2out key add failed");
711 /* Update IP checksum */
713 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
714 ip->checksum = ip_csum_fold (sum);
716 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
721 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
722 /* Per-user LRU list maintenance */
723 clib_dlist_remove (tsm->list_pool, s->per_user_index);
724 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
730 static snat_session_t *
731 snat_out2in_lb (snat_main_t *sm,
738 vlib_node_runtime_t * node)
740 nat_ed_ses_key_t key;
741 clib_bihash_kv_16_8_t s_kv, s_value;
742 udp_header_t *udp = ip4_next_header (ip);
743 tcp_header_t *tcp = (tcp_header_t *) udp;
744 snat_session_t *s = 0;
745 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
746 snat_session_key_t e_key, l_key;
747 u32 old_addr, new_addr;
748 u32 proto = ip_proto_to_snat_proto (ip->protocol);
749 u16 new_port, old_port;
753 snat_session_key_t eh_key;
756 old_addr = ip->dst_address.as_u32;
758 key.l_addr = ip->dst_address;
759 key.r_addr = ip->src_address;
760 key.fib_index = rx_fib_index;
761 key.proto = ip->protocol;
762 key.r_port = udp->src_port;
763 key.l_port = udp->dst_port;
764 s_kv.key[0] = key.as_u64[0];
765 s_kv.key[1] = key.as_u64[1];
767 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
769 s = pool_elt_at_index (tsm->sessions, s_value.value);
773 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
775 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
779 e_key.addr = ip->dst_address;
780 e_key.port = udp->dst_port;
781 e_key.protocol = proto;
782 e_key.fib_index = rx_fib_index;
783 if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0, &twice_nat))
786 u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index,
790 clib_warning ("create NAT user failed");
794 s = nat_session_alloc_or_recycle (sm, u, thread_index);
797 clib_warning ("create NAT session failed");
801 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
802 s->ext_host_port = udp->src_port;
803 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
804 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
805 s->outside_address_index = ~0;
808 u->nstaticsessions++;
810 /* Add to lookup tables */
811 s_kv.value = s - tsm->sessions;
812 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
813 clib_warning ("out2in-ed key add failed");
817 eh_key.protocol = proto;
818 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
819 thread_index, &eh_key,
822 sm->per_thread_data[thread_index].snat_thread_index))
824 b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
827 key.r_addr.as_u32 = s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
828 key.r_port = s->ext_host_nat_port = eh_key.port;
829 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
831 key.l_addr = l_key.addr;
832 key.fib_index = l_key.fib_index;
833 key.l_port = l_key.port;
834 s_kv.key[0] = key.as_u64[0];
835 s_kv.key[1] = key.as_u64[1];
836 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
837 clib_warning ("in2out-ed key add failed");
840 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
842 /* Update IP checksum */
844 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
845 if (is_twice_nat_session (s))
846 sum = ip_csum_update (sum, ip->src_address.as_u32,
847 s->ext_host_nat_addr.as_u32, ip4_header_t,
849 ip->checksum = ip_csum_fold (sum);
851 if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
853 old_port = tcp->dst_port;
854 tcp->dst_port = s->in2out.port;
855 new_port = tcp->dst_port;
858 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
859 sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
860 if (is_twice_nat_session (s))
862 sum = ip_csum_update (sum, ip->src_address.as_u32,
863 s->ext_host_nat_addr.as_u32, ip4_header_t,
865 sum = ip_csum_update (sum, tcp->src_port, s->ext_host_nat_port,
866 ip4_header_t, length);
867 tcp->src_port = s->ext_host_nat_port;
868 ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
870 tcp->checksum = ip_csum_fold(sum);
874 udp->dst_port = s->in2out.port;
875 if (is_twice_nat_session (s))
877 udp->src_port = s->ext_host_nat_port;
878 ip->src_address.as_u32 = s->ext_host_nat_addr.as_u32;
883 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
888 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
889 /* Per-user LRU list maintenance */
890 clib_dlist_remove (tsm->list_pool, s->per_user_index);
891 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
898 snat_out2in_node_fn (vlib_main_t * vm,
899 vlib_node_runtime_t * node,
900 vlib_frame_t * frame)
902 u32 n_left_from, * from, * to_next;
903 snat_out2in_next_t next_index;
904 u32 pkts_processed = 0;
905 snat_main_t * sm = &snat_main;
906 f64 now = vlib_time_now (vm);
907 u32 thread_index = vlib_get_thread_index ();
909 from = vlib_frame_vector_args (frame);
910 n_left_from = frame->n_vectors;
911 next_index = node->cached_next_index;
913 while (n_left_from > 0)
917 vlib_get_next_frame (vm, node, next_index,
918 to_next, n_left_to_next);
920 while (n_left_from >= 4 && n_left_to_next >= 2)
923 vlib_buffer_t * b0, * b1;
924 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
925 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
926 u32 sw_if_index0, sw_if_index1;
927 ip4_header_t * ip0, *ip1;
928 ip_csum_t sum0, sum1;
929 u32 new_addr0, old_addr0;
930 u16 new_port0, old_port0;
931 u32 new_addr1, old_addr1;
932 u16 new_port1, old_port1;
933 udp_header_t * udp0, * udp1;
934 tcp_header_t * tcp0, * tcp1;
935 icmp46_header_t * icmp0, * icmp1;
936 snat_session_key_t key0, key1, sm0, sm1;
937 u32 rx_fib_index0, rx_fib_index1;
939 snat_session_t * s0 = 0, * s1 = 0;
940 clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
942 /* Prefetch next iteration. */
944 vlib_buffer_t * p2, * p3;
946 p2 = vlib_get_buffer (vm, from[2]);
947 p3 = vlib_get_buffer (vm, from[3]);
949 vlib_prefetch_buffer_header (p2, LOAD);
950 vlib_prefetch_buffer_header (p3, LOAD);
952 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
953 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
956 /* speculatively enqueue b0 and b1 to the current next frame */
957 to_next[0] = bi0 = from[0];
958 to_next[1] = bi1 = from[1];
964 b0 = vlib_get_buffer (vm, bi0);
965 b1 = vlib_get_buffer (vm, bi1);
967 vnet_buffer (b0)->snat.flags = 0;
968 vnet_buffer (b1)->snat.flags = 0;
970 ip0 = vlib_buffer_get_current (b0);
971 udp0 = ip4_next_header (ip0);
972 tcp0 = (tcp_header_t *) udp0;
973 icmp0 = (icmp46_header_t *) udp0;
975 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
976 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
979 if (PREDICT_FALSE(ip0->ttl == 1))
981 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
982 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
983 ICMP4_time_exceeded_ttl_exceeded_in_transit,
985 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
989 proto0 = ip_proto_to_snat_proto (ip0->protocol);
991 if (PREDICT_FALSE (proto0 == ~0))
993 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
994 thread_index, now, vm, node);
996 next0 = SNAT_OUT2IN_NEXT_DROP;
1000 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1002 next0 = icmp_out2in_slow_path
1003 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1004 next0, now, thread_index, &s0);
1008 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1010 next0 = SNAT_OUT2IN_NEXT_REASS;
1014 key0.addr = ip0->dst_address;
1015 key0.port = udp0->dst_port;
1016 key0.protocol = proto0;
1017 key0.fib_index = rx_fib_index0;
1019 kv0.key = key0.as_u64;
1021 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1024 /* Try to match static mapping by external address and port,
1025 destination address and port in packet */
1026 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1028 if (!sm->forwarding_enabled)
1030 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1032 * Send DHCP packets to the ipv4 stack, or we won't
1033 * be able to use dhcp client on the outside interface
1035 if (proto0 != SNAT_PROTOCOL_UDP
1037 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1038 next0 = SNAT_OUT2IN_NEXT_DROP;
1045 /* Create session initiated by host from external network */
1046 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1050 next0 = SNAT_OUT2IN_NEXT_DROP;
1056 if (PREDICT_FALSE (value0.value == ~0ULL))
1058 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1061 next0 = SNAT_OUT2IN_NEXT_DROP;
1066 s0 = pool_elt_at_index (
1067 sm->per_thread_data[thread_index].sessions,
1072 old_addr0 = ip0->dst_address.as_u32;
1073 ip0->dst_address = s0->in2out.addr;
1074 new_addr0 = ip0->dst_address.as_u32;
1075 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1077 sum0 = ip0->checksum;
1078 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1080 dst_address /* changed member */);
1081 ip0->checksum = ip_csum_fold (sum0);
1083 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1085 old_port0 = tcp0->dst_port;
1086 tcp0->dst_port = s0->in2out.port;
1087 new_port0 = tcp0->dst_port;
1089 sum0 = tcp0->checksum;
1090 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1092 dst_address /* changed member */);
1094 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1095 ip4_header_t /* cheat */,
1096 length /* changed member */);
1097 tcp0->checksum = ip_csum_fold(sum0);
1101 old_port0 = udp0->dst_port;
1102 udp0->dst_port = s0->in2out.port;
1107 s0->last_heard = now;
1109 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1110 /* Per-user LRU list maintenance */
1111 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1112 s0->per_user_index);
1113 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1114 s0->per_user_list_head_index,
1115 s0->per_user_index);
1118 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1119 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1121 snat_out2in_trace_t *t =
1122 vlib_add_trace (vm, node, b0, sizeof (*t));
1123 t->sw_if_index = sw_if_index0;
1124 t->next_index = next0;
1125 t->session_index = ~0;
1127 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1130 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1133 ip1 = vlib_buffer_get_current (b1);
1134 udp1 = ip4_next_header (ip1);
1135 tcp1 = (tcp_header_t *) udp1;
1136 icmp1 = (icmp46_header_t *) udp1;
1138 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1139 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1142 if (PREDICT_FALSE(ip1->ttl == 1))
1144 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1145 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1146 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1148 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1152 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1154 if (PREDICT_FALSE (proto1 == ~0))
1156 s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1157 thread_index, now, vm, node);
1159 next1 = SNAT_OUT2IN_NEXT_DROP;
1163 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1165 next1 = icmp_out2in_slow_path
1166 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1167 next1, now, thread_index, &s1);
1171 if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1173 next1 = SNAT_OUT2IN_NEXT_REASS;
1177 key1.addr = ip1->dst_address;
1178 key1.port = udp1->dst_port;
1179 key1.protocol = proto1;
1180 key1.fib_index = rx_fib_index1;
1182 kv1.key = key1.as_u64;
1184 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1187 /* Try to match static mapping by external address and port,
1188 destination address and port in packet */
1189 if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0))
1191 if (!sm->forwarding_enabled)
1193 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1195 * Send DHCP packets to the ipv4 stack, or we won't
1196 * be able to use dhcp client on the outside interface
1198 if (proto1 != SNAT_PROTOCOL_UDP
1200 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1201 next1 = SNAT_OUT2IN_NEXT_DROP;
1208 /* Create session initiated by host from external network */
1209 s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1213 next1 = SNAT_OUT2IN_NEXT_DROP;
1219 if (PREDICT_FALSE (value1.value == ~0ULL))
1221 s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1224 next1 = SNAT_OUT2IN_NEXT_DROP;
1229 s1 = pool_elt_at_index (
1230 sm->per_thread_data[thread_index].sessions,
1235 old_addr1 = ip1->dst_address.as_u32;
1236 ip1->dst_address = s1->in2out.addr;
1237 new_addr1 = ip1->dst_address.as_u32;
1238 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1240 sum1 = ip1->checksum;
1241 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1243 dst_address /* changed member */);
1244 ip1->checksum = ip_csum_fold (sum1);
1246 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1248 old_port1 = tcp1->dst_port;
1249 tcp1->dst_port = s1->in2out.port;
1250 new_port1 = tcp1->dst_port;
1252 sum1 = tcp1->checksum;
1253 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1255 dst_address /* changed member */);
1257 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1258 ip4_header_t /* cheat */,
1259 length /* changed member */);
1260 tcp1->checksum = ip_csum_fold(sum1);
1264 old_port1 = udp1->dst_port;
1265 udp1->dst_port = s1->in2out.port;
1270 s1->last_heard = now;
1272 s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1273 /* Per-user LRU list maintenance */
1274 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1275 s1->per_user_index);
1276 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1277 s1->per_user_list_head_index,
1278 s1->per_user_index);
1281 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1282 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1284 snat_out2in_trace_t *t =
1285 vlib_add_trace (vm, node, b1, sizeof (*t));
1286 t->sw_if_index = sw_if_index1;
1287 t->next_index = next1;
1288 t->session_index = ~0;
1290 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1293 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1295 /* verify speculative enqueues, maybe switch current next frame */
1296 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1297 to_next, n_left_to_next,
1298 bi0, bi1, next0, next1);
1301 while (n_left_from > 0 && n_left_to_next > 0)
1305 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1309 u32 new_addr0, old_addr0;
1310 u16 new_port0, old_port0;
1311 udp_header_t * udp0;
1312 tcp_header_t * tcp0;
1313 icmp46_header_t * icmp0;
1314 snat_session_key_t key0, sm0;
1317 snat_session_t * s0 = 0;
1318 clib_bihash_kv_8_8_t kv0, value0;
1320 /* speculatively enqueue b0 to the current next frame */
1326 n_left_to_next -= 1;
1328 b0 = vlib_get_buffer (vm, bi0);
1330 vnet_buffer (b0)->snat.flags = 0;
1332 ip0 = vlib_buffer_get_current (b0);
1333 udp0 = ip4_next_header (ip0);
1334 tcp0 = (tcp_header_t *) udp0;
1335 icmp0 = (icmp46_header_t *) udp0;
1337 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1338 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1341 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1343 if (PREDICT_FALSE (proto0 == ~0))
1345 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1346 thread_index, now, vm, node);
1348 next0 = SNAT_OUT2IN_NEXT_DROP;
1352 if (PREDICT_FALSE(ip0->ttl == 1))
1354 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1355 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1356 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1358 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1362 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1364 next0 = icmp_out2in_slow_path
1365 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1366 next0, now, thread_index, &s0);
1370 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1372 next0 = SNAT_OUT2IN_NEXT_REASS;
1376 key0.addr = ip0->dst_address;
1377 key0.port = udp0->dst_port;
1378 key0.protocol = proto0;
1379 key0.fib_index = rx_fib_index0;
1381 kv0.key = key0.as_u64;
1383 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1386 /* Try to match static mapping by external address and port,
1387 destination address and port in packet */
1388 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1390 if (!sm->forwarding_enabled)
1392 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1394 * Send DHCP packets to the ipv4 stack, or we won't
1395 * be able to use dhcp client on the outside interface
1397 if (proto0 != SNAT_PROTOCOL_UDP
1399 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1400 next0 = SNAT_OUT2IN_NEXT_DROP;
1407 /* Create session initiated by host from external network */
1408 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1412 next0 = SNAT_OUT2IN_NEXT_DROP;
1418 if (PREDICT_FALSE (value0.value == ~0ULL))
1420 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1423 next0 = SNAT_OUT2IN_NEXT_DROP;
1428 s0 = pool_elt_at_index (
1429 sm->per_thread_data[thread_index].sessions,
1434 old_addr0 = ip0->dst_address.as_u32;
1435 ip0->dst_address = s0->in2out.addr;
1436 new_addr0 = ip0->dst_address.as_u32;
1437 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1439 sum0 = ip0->checksum;
1440 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1442 dst_address /* changed member */);
1443 ip0->checksum = ip_csum_fold (sum0);
1445 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1447 old_port0 = tcp0->dst_port;
1448 tcp0->dst_port = s0->in2out.port;
1449 new_port0 = tcp0->dst_port;
1451 sum0 = tcp0->checksum;
1452 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1454 dst_address /* changed member */);
1456 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1457 ip4_header_t /* cheat */,
1458 length /* changed member */);
1459 tcp0->checksum = ip_csum_fold(sum0);
1463 old_port0 = udp0->dst_port;
1464 udp0->dst_port = s0->in2out.port;
1469 s0->last_heard = now;
1471 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1472 /* Per-user LRU list maintenance */
1473 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1474 s0->per_user_index);
1475 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1476 s0->per_user_list_head_index,
1477 s0->per_user_index);
1480 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1481 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1483 snat_out2in_trace_t *t =
1484 vlib_add_trace (vm, node, b0, sizeof (*t));
1485 t->sw_if_index = sw_if_index0;
1486 t->next_index = next0;
1487 t->session_index = ~0;
1489 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1492 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1494 /* verify speculative enqueue, maybe switch current next frame */
1495 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1496 to_next, n_left_to_next,
1500 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1503 vlib_node_increment_counter (vm, snat_out2in_node.index,
1504 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1506 return frame->n_vectors;
1509 VLIB_REGISTER_NODE (snat_out2in_node) = {
1510 .function = snat_out2in_node_fn,
1511 .name = "nat44-out2in",
1512 .vector_size = sizeof (u32),
1513 .format_trace = format_snat_out2in_trace,
1514 .type = VLIB_NODE_TYPE_INTERNAL,
1516 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1517 .error_strings = snat_out2in_error_strings,
1519 .runtime_data_bytes = sizeof (snat_runtime_t),
1521 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1523 /* edit / add dispositions here */
1525 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1526 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1527 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1528 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1531 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1534 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1535 vlib_node_runtime_t * node,
1536 vlib_frame_t * frame)
1538 u32 n_left_from, *from, *to_next;
1539 snat_out2in_next_t next_index;
1540 u32 pkts_processed = 0;
1541 snat_main_t *sm = &snat_main;
1542 f64 now = vlib_time_now (vm);
1543 u32 thread_index = vlib_get_thread_index ();
1544 snat_main_per_thread_data_t *per_thread_data =
1545 &sm->per_thread_data[thread_index];
1546 u32 *fragments_to_drop = 0;
1547 u32 *fragments_to_loopback = 0;
1549 from = vlib_frame_vector_args (frame);
1550 n_left_from = frame->n_vectors;
1551 next_index = node->cached_next_index;
1553 while (n_left_from > 0)
1557 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1559 while (n_left_from > 0 && n_left_to_next > 0)
1561 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1566 nat_reass_ip4_t *reass0;
1567 udp_header_t * udp0;
1568 tcp_header_t * tcp0;
1569 snat_session_key_t key0, sm0;
1570 clib_bihash_kv_8_8_t kv0, value0;
1571 snat_session_t * s0 = 0;
1572 u16 old_port0, new_port0;
1575 /* speculatively enqueue b0 to the current next frame */
1581 n_left_to_next -= 1;
1583 b0 = vlib_get_buffer (vm, bi0);
1584 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1586 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1587 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1590 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1592 next0 = SNAT_OUT2IN_NEXT_DROP;
1593 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1597 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1598 udp0 = ip4_next_header (ip0);
1599 tcp0 = (tcp_header_t *) udp0;
1600 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1602 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1607 &fragments_to_drop);
1609 if (PREDICT_FALSE (!reass0))
1611 next0 = SNAT_OUT2IN_NEXT_DROP;
1612 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1616 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1618 key0.addr = ip0->dst_address;
1619 key0.port = udp0->dst_port;
1620 key0.protocol = proto0;
1621 key0.fib_index = rx_fib_index0;
1622 kv0.key = key0.as_u64;
1624 if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1626 /* Try to match static mapping by external address and port,
1627 destination address and port in packet */
1628 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1630 if (!sm->forwarding_enabled)
1632 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1634 * Send DHCP packets to the ipv4 stack, or we won't
1635 * be able to use dhcp client on the outside interface
1637 if (proto0 != SNAT_PROTOCOL_UDP
1639 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1640 next0 = SNAT_OUT2IN_NEXT_DROP;
1647 /* Create session initiated by host from external network */
1648 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1652 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1653 next0 = SNAT_OUT2IN_NEXT_DROP;
1656 reass0->sess_index = s0 - per_thread_data->sessions;
1657 reass0->thread_index = thread_index;
1661 s0 = pool_elt_at_index (per_thread_data->sessions,
1663 reass0->sess_index = value0.value;
1665 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1669 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1671 if (nat_ip4_reass_add_fragment (reass0, bi0))
1673 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1674 next0 = SNAT_OUT2IN_NEXT_DROP;
1680 s0 = pool_elt_at_index (per_thread_data->sessions,
1681 reass0->sess_index);
1684 old_addr0 = ip0->dst_address.as_u32;
1685 ip0->dst_address = s0->in2out.addr;
1686 new_addr0 = ip0->dst_address.as_u32;
1687 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1689 sum0 = ip0->checksum;
1690 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1692 dst_address /* changed member */);
1693 ip0->checksum = ip_csum_fold (sum0);
1695 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1697 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1699 old_port0 = tcp0->dst_port;
1700 tcp0->dst_port = s0->in2out.port;
1701 new_port0 = tcp0->dst_port;
1703 sum0 = tcp0->checksum;
1704 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1706 dst_address /* changed member */);
1708 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1709 ip4_header_t /* cheat */,
1710 length /* changed member */);
1711 tcp0->checksum = ip_csum_fold(sum0);
1715 old_port0 = udp0->dst_port;
1716 udp0->dst_port = s0->in2out.port;
1722 s0->last_heard = now;
1724 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1725 /* Per-user LRU list maintenance */
1726 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1727 s0->per_user_index);
1728 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1729 s0->per_user_list_head_index,
1730 s0->per_user_index);
1733 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1734 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1736 nat44_out2in_reass_trace_t *t =
1737 vlib_add_trace (vm, node, b0, sizeof (*t));
1738 t->cached = cached0;
1739 t->sw_if_index = sw_if_index0;
1740 t->next_index = next0;
1750 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1752 /* verify speculative enqueue, maybe switch current next frame */
1753 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1754 to_next, n_left_to_next,
1758 if (n_left_from == 0 && vec_len (fragments_to_loopback))
1760 from = vlib_frame_vector_args (frame);
1761 u32 len = vec_len (fragments_to_loopback);
1762 if (len <= VLIB_FRAME_SIZE)
1764 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1766 vec_reset_length (fragments_to_loopback);
1771 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1772 sizeof (u32) * VLIB_FRAME_SIZE);
1773 n_left_from = VLIB_FRAME_SIZE;
1774 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1779 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1782 vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1783 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1786 nat_send_all_to_node (vm, fragments_to_drop, node,
1787 &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1788 SNAT_OUT2IN_NEXT_DROP);
1790 vec_free (fragments_to_drop);
1791 vec_free (fragments_to_loopback);
1792 return frame->n_vectors;
1795 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1796 .function = nat44_out2in_reass_node_fn,
1797 .name = "nat44-out2in-reass",
1798 .vector_size = sizeof (u32),
1799 .format_trace = format_nat44_out2in_reass_trace,
1800 .type = VLIB_NODE_TYPE_INTERNAL,
1802 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1803 .error_strings = snat_out2in_error_strings,
1805 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1807 /* edit / add dispositions here */
1809 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1810 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1811 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1812 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1815 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1816 nat44_out2in_reass_node_fn);
1818 /**************************/
1819 /*** deterministic mode ***/
1820 /**************************/
1822 snat_det_out2in_node_fn (vlib_main_t * vm,
1823 vlib_node_runtime_t * node,
1824 vlib_frame_t * frame)
1826 u32 n_left_from, * from, * to_next;
1827 snat_out2in_next_t next_index;
1828 u32 pkts_processed = 0;
1829 snat_main_t * sm = &snat_main;
1830 u32 thread_index = vlib_get_thread_index ();
1832 from = vlib_frame_vector_args (frame);
1833 n_left_from = frame->n_vectors;
1834 next_index = node->cached_next_index;
1836 while (n_left_from > 0)
1840 vlib_get_next_frame (vm, node, next_index,
1841 to_next, n_left_to_next);
1843 while (n_left_from >= 4 && n_left_to_next >= 2)
1846 vlib_buffer_t * b0, * b1;
1847 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1848 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1849 u32 sw_if_index0, sw_if_index1;
1850 ip4_header_t * ip0, * ip1;
1851 ip_csum_t sum0, sum1;
1852 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1853 u16 new_port0, old_port0, old_port1, new_port1;
1854 udp_header_t * udp0, * udp1;
1855 tcp_header_t * tcp0, * tcp1;
1857 snat_det_out_key_t key0, key1;
1858 snat_det_map_t * dm0, * dm1;
1859 snat_det_session_t * ses0 = 0, * ses1 = 0;
1860 u32 rx_fib_index0, rx_fib_index1;
1861 icmp46_header_t * icmp0, * icmp1;
1863 /* Prefetch next iteration. */
1865 vlib_buffer_t * p2, * p3;
1867 p2 = vlib_get_buffer (vm, from[2]);
1868 p3 = vlib_get_buffer (vm, from[3]);
1870 vlib_prefetch_buffer_header (p2, LOAD);
1871 vlib_prefetch_buffer_header (p3, LOAD);
1873 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1874 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1877 /* speculatively enqueue b0 and b1 to the current next frame */
1878 to_next[0] = bi0 = from[0];
1879 to_next[1] = bi1 = from[1];
1883 n_left_to_next -= 2;
1885 b0 = vlib_get_buffer (vm, bi0);
1886 b1 = vlib_get_buffer (vm, bi1);
1888 ip0 = vlib_buffer_get_current (b0);
1889 udp0 = ip4_next_header (ip0);
1890 tcp0 = (tcp_header_t *) udp0;
1892 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1894 if (PREDICT_FALSE(ip0->ttl == 1))
1896 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1897 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1898 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1900 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1904 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1906 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1908 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1909 icmp0 = (icmp46_header_t *) udp0;
1911 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1912 rx_fib_index0, node, next0, thread_index,
1917 key0.ext_host_addr = ip0->src_address;
1918 key0.ext_host_port = tcp0->src;
1919 key0.out_port = tcp0->dst;
1921 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1922 if (PREDICT_FALSE(!dm0))
1924 clib_warning("unknown dst address: %U",
1925 format_ip4_address, &ip0->dst_address);
1926 next0 = SNAT_OUT2IN_NEXT_DROP;
1927 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1931 snat_det_reverse(dm0, &ip0->dst_address,
1932 clib_net_to_host_u16(tcp0->dst), &new_addr0);
1934 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1935 if (PREDICT_FALSE(!ses0))
1937 clib_warning("no match src %U:%d dst %U:%d for user %U",
1938 format_ip4_address, &ip0->src_address,
1939 clib_net_to_host_u16 (tcp0->src),
1940 format_ip4_address, &ip0->dst_address,
1941 clib_net_to_host_u16 (tcp0->dst),
1942 format_ip4_address, &new_addr0);
1943 next0 = SNAT_OUT2IN_NEXT_DROP;
1944 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1947 new_port0 = ses0->in_port;
1949 old_addr0 = ip0->dst_address;
1950 ip0->dst_address = new_addr0;
1951 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1953 sum0 = ip0->checksum;
1954 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1956 dst_address /* changed member */);
1957 ip0->checksum = ip_csum_fold (sum0);
1959 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1961 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1962 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1963 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1964 snat_det_ses_close(dm0, ses0);
1966 old_port0 = tcp0->dst;
1967 tcp0->dst = new_port0;
1969 sum0 = tcp0->checksum;
1970 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1972 dst_address /* changed member */);
1974 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1975 ip4_header_t /* cheat */,
1976 length /* changed member */);
1977 tcp0->checksum = ip_csum_fold(sum0);
1981 old_port0 = udp0->dst_port;
1982 udp0->dst_port = new_port0;
1988 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1989 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1991 snat_out2in_trace_t *t =
1992 vlib_add_trace (vm, node, b0, sizeof (*t));
1993 t->sw_if_index = sw_if_index0;
1994 t->next_index = next0;
1995 t->session_index = ~0;
1997 t->session_index = ses0 - dm0->sessions;
2000 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2002 b1 = vlib_get_buffer (vm, bi1);
2004 ip1 = vlib_buffer_get_current (b1);
2005 udp1 = ip4_next_header (ip1);
2006 tcp1 = (tcp_header_t *) udp1;
2008 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2010 if (PREDICT_FALSE(ip1->ttl == 1))
2012 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2013 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2014 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2016 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2020 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2022 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2024 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2025 icmp1 = (icmp46_header_t *) udp1;
2027 next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2028 rx_fib_index1, node, next1, thread_index,
2033 key1.ext_host_addr = ip1->src_address;
2034 key1.ext_host_port = tcp1->src;
2035 key1.out_port = tcp1->dst;
2037 dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2038 if (PREDICT_FALSE(!dm1))
2040 clib_warning("unknown dst address: %U",
2041 format_ip4_address, &ip1->dst_address);
2042 next1 = SNAT_OUT2IN_NEXT_DROP;
2043 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2047 snat_det_reverse(dm1, &ip1->dst_address,
2048 clib_net_to_host_u16(tcp1->dst), &new_addr1);
2050 ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2051 if (PREDICT_FALSE(!ses1))
2053 clib_warning("no match src %U:%d dst %U:%d for user %U",
2054 format_ip4_address, &ip1->src_address,
2055 clib_net_to_host_u16 (tcp1->src),
2056 format_ip4_address, &ip1->dst_address,
2057 clib_net_to_host_u16 (tcp1->dst),
2058 format_ip4_address, &new_addr1);
2059 next1 = SNAT_OUT2IN_NEXT_DROP;
2060 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2063 new_port1 = ses1->in_port;
2065 old_addr1 = ip1->dst_address;
2066 ip1->dst_address = new_addr1;
2067 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2069 sum1 = ip1->checksum;
2070 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2072 dst_address /* changed member */);
2073 ip1->checksum = ip_csum_fold (sum1);
2075 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2077 if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2078 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2079 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2080 snat_det_ses_close(dm1, ses1);
2082 old_port1 = tcp1->dst;
2083 tcp1->dst = new_port1;
2085 sum1 = tcp1->checksum;
2086 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2088 dst_address /* changed member */);
2090 sum1 = ip_csum_update (sum1, old_port1, new_port1,
2091 ip4_header_t /* cheat */,
2092 length /* changed member */);
2093 tcp1->checksum = ip_csum_fold(sum1);
2097 old_port1 = udp1->dst_port;
2098 udp1->dst_port = new_port1;
2104 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2105 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2107 snat_out2in_trace_t *t =
2108 vlib_add_trace (vm, node, b1, sizeof (*t));
2109 t->sw_if_index = sw_if_index1;
2110 t->next_index = next1;
2111 t->session_index = ~0;
2113 t->session_index = ses1 - dm1->sessions;
2116 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2118 /* verify speculative enqueues, maybe switch current next frame */
2119 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2120 to_next, n_left_to_next,
2121 bi0, bi1, next0, next1);
2124 while (n_left_from > 0 && n_left_to_next > 0)
2128 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2132 ip4_address_t new_addr0, old_addr0;
2133 u16 new_port0, old_port0;
2134 udp_header_t * udp0;
2135 tcp_header_t * tcp0;
2137 snat_det_out_key_t key0;
2138 snat_det_map_t * dm0;
2139 snat_det_session_t * ses0 = 0;
2141 icmp46_header_t * icmp0;
2143 /* speculatively enqueue b0 to the current next frame */
2149 n_left_to_next -= 1;
2151 b0 = vlib_get_buffer (vm, bi0);
2153 ip0 = vlib_buffer_get_current (b0);
2154 udp0 = ip4_next_header (ip0);
2155 tcp0 = (tcp_header_t *) udp0;
2157 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2159 if (PREDICT_FALSE(ip0->ttl == 1))
2161 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2162 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2163 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2165 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2169 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2171 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2173 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2174 icmp0 = (icmp46_header_t *) udp0;
2176 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2177 rx_fib_index0, node, next0, thread_index,
2182 key0.ext_host_addr = ip0->src_address;
2183 key0.ext_host_port = tcp0->src;
2184 key0.out_port = tcp0->dst;
2186 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2187 if (PREDICT_FALSE(!dm0))
2189 clib_warning("unknown dst address: %U",
2190 format_ip4_address, &ip0->dst_address);
2191 next0 = SNAT_OUT2IN_NEXT_DROP;
2192 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2196 snat_det_reverse(dm0, &ip0->dst_address,
2197 clib_net_to_host_u16(tcp0->dst), &new_addr0);
2199 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2200 if (PREDICT_FALSE(!ses0))
2202 clib_warning("no match src %U:%d dst %U:%d for user %U",
2203 format_ip4_address, &ip0->src_address,
2204 clib_net_to_host_u16 (tcp0->src),
2205 format_ip4_address, &ip0->dst_address,
2206 clib_net_to_host_u16 (tcp0->dst),
2207 format_ip4_address, &new_addr0);
2208 next0 = SNAT_OUT2IN_NEXT_DROP;
2209 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2212 new_port0 = ses0->in_port;
2214 old_addr0 = ip0->dst_address;
2215 ip0->dst_address = new_addr0;
2216 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2218 sum0 = ip0->checksum;
2219 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2221 dst_address /* changed member */);
2222 ip0->checksum = ip_csum_fold (sum0);
2224 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2226 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2227 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2228 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2229 snat_det_ses_close(dm0, ses0);
2231 old_port0 = tcp0->dst;
2232 tcp0->dst = new_port0;
2234 sum0 = tcp0->checksum;
2235 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2237 dst_address /* changed member */);
2239 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2240 ip4_header_t /* cheat */,
2241 length /* changed member */);
2242 tcp0->checksum = ip_csum_fold(sum0);
2246 old_port0 = udp0->dst_port;
2247 udp0->dst_port = new_port0;
2253 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2254 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2256 snat_out2in_trace_t *t =
2257 vlib_add_trace (vm, node, b0, sizeof (*t));
2258 t->sw_if_index = sw_if_index0;
2259 t->next_index = next0;
2260 t->session_index = ~0;
2262 t->session_index = ses0 - dm0->sessions;
2265 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2267 /* verify speculative enqueue, maybe switch current next frame */
2268 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2269 to_next, n_left_to_next,
2273 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2276 vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2277 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2279 return frame->n_vectors;
2282 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2283 .function = snat_det_out2in_node_fn,
2284 .name = "nat44-det-out2in",
2285 .vector_size = sizeof (u32),
2286 .format_trace = format_snat_out2in_trace,
2287 .type = VLIB_NODE_TYPE_INTERNAL,
2289 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2290 .error_strings = snat_out2in_error_strings,
2292 .runtime_data_bytes = sizeof (snat_runtime_t),
2294 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2296 /* edit / add dispositions here */
2298 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2299 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2300 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2301 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2304 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2307 * Get address and port values to be used for ICMP packet translation
2308 * and create session if needed
2310 * @param[in,out] sm NAT main
2311 * @param[in,out] node NAT node runtime
2312 * @param[in] thread_index thread index
2313 * @param[in,out] b0 buffer containing packet to be translated
2314 * @param[out] p_proto protocol used for matching
2315 * @param[out] p_value address and port after NAT translation
2316 * @param[out] p_dont_translate if packet should not be translated
2317 * @param d optional parameter
2318 * @param e optional parameter
2320 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2321 u32 thread_index, vlib_buffer_t *b0,
2322 ip4_header_t *ip0, u8 *p_proto,
2323 snat_session_key_t *p_value,
2324 u8 *p_dont_translate, void *d, void *e)
2326 icmp46_header_t *icmp0;
2329 snat_det_out_key_t key0;
2330 u8 dont_translate = 0;
2332 icmp_echo_header_t *echo0, *inner_echo0 = 0;
2333 ip4_header_t *inner_ip0;
2334 void *l4_header = 0;
2335 icmp46_header_t *inner_icmp0;
2336 snat_det_map_t * dm0 = 0;
2337 ip4_address_t new_addr0 = {{0}};
2338 snat_det_session_t * ses0 = 0;
2339 ip4_address_t out_addr;
2341 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2342 echo0 = (icmp_echo_header_t *)(icmp0+1);
2343 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2345 if (!icmp_is_error_message (icmp0))
2347 protocol = SNAT_PROTOCOL_ICMP;
2348 key0.ext_host_addr = ip0->src_address;
2349 key0.ext_host_port = 0;
2350 key0.out_port = echo0->identifier;
2351 out_addr = ip0->dst_address;
2355 inner_ip0 = (ip4_header_t *)(echo0+1);
2356 l4_header = ip4_next_header (inner_ip0);
2357 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2358 key0.ext_host_addr = inner_ip0->dst_address;
2359 out_addr = inner_ip0->src_address;
2362 case SNAT_PROTOCOL_ICMP:
2363 inner_icmp0 = (icmp46_header_t*)l4_header;
2364 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2365 key0.ext_host_port = 0;
2366 key0.out_port = inner_echo0->identifier;
2368 case SNAT_PROTOCOL_UDP:
2369 case SNAT_PROTOCOL_TCP:
2370 key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2371 key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2374 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2375 next0 = SNAT_OUT2IN_NEXT_DROP;
2380 dm0 = snat_det_map_by_out(sm, &out_addr);
2381 if (PREDICT_FALSE(!dm0))
2383 /* Don't NAT packet aimed at the intfc address */
2384 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2385 ip0->dst_address.as_u32)))
2390 clib_warning("unknown dst address: %U",
2391 format_ip4_address, &ip0->dst_address);
2395 snat_det_reverse(dm0, &ip0->dst_address,
2396 clib_net_to_host_u16(key0.out_port), &new_addr0);
2398 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2399 if (PREDICT_FALSE(!ses0))
2401 /* Don't NAT packet aimed at the intfc address */
2402 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2403 ip0->dst_address.as_u32)))
2408 clib_warning("no match src %U:%d dst %U:%d for user %U",
2409 format_ip4_address, &key0.ext_host_addr,
2410 clib_net_to_host_u16 (key0.ext_host_port),
2411 format_ip4_address, &out_addr,
2412 clib_net_to_host_u16 (key0.out_port),
2413 format_ip4_address, &new_addr0);
2414 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2415 next0 = SNAT_OUT2IN_NEXT_DROP;
2419 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2420 !icmp_is_error_message (icmp0)))
2422 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2423 next0 = SNAT_OUT2IN_NEXT_DROP;
2430 *p_proto = protocol;
2433 p_value->addr = new_addr0;
2434 p_value->fib_index = sm->inside_fib_index;
2435 p_value->port = ses0->in_port;
2437 *p_dont_translate = dont_translate;
2439 *(snat_det_session_t**)d = ses0;
2441 *(snat_det_map_t**)e = dm0;
2445 /**********************/
2446 /*** worker handoff ***/
2447 /**********************/
2449 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2450 vlib_node_runtime_t * node,
2451 vlib_frame_t * frame)
2453 snat_main_t *sm = &snat_main;
2454 vlib_thread_main_t *tm = vlib_get_thread_main ();
2455 u32 n_left_from, *from, *to_next = 0;
2456 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2457 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2459 vlib_frame_queue_elt_t *hf = 0;
2460 vlib_frame_t *f = 0;
2462 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2463 u32 next_worker_index = 0;
2464 u32 current_worker_index = ~0;
2465 u32 thread_index = vlib_get_thread_index ();
2467 ASSERT (vec_len (sm->workers));
2469 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2471 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2473 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2474 sm->first_worker_index + sm->num_workers - 1,
2475 (vlib_frame_queue_t *) (~0));
2478 from = vlib_frame_vector_args (frame);
2479 n_left_from = frame->n_vectors;
2481 while (n_left_from > 0)
2494 b0 = vlib_get_buffer (vm, bi0);
2496 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2497 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2499 ip0 = vlib_buffer_get_current (b0);
2501 next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2503 if (PREDICT_FALSE (next_worker_index != thread_index))
2507 if (next_worker_index != current_worker_index)
2510 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2512 hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2514 handoff_queue_elt_by_worker_index);
2516 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2517 to_next_worker = &hf->buffer_index[hf->n_vectors];
2518 current_worker_index = next_worker_index;
2521 /* enqueue to correct worker thread */
2522 to_next_worker[0] = bi0;
2524 n_left_to_next_worker--;
2526 if (n_left_to_next_worker == 0)
2528 hf->n_vectors = VLIB_FRAME_SIZE;
2529 vlib_put_frame_queue_elt (hf);
2530 current_worker_index = ~0;
2531 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2538 /* if this is 1st frame */
2541 f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2542 to_next = vlib_frame_vector_args (f);
2550 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2551 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2553 snat_out2in_worker_handoff_trace_t *t =
2554 vlib_add_trace (vm, node, b0, sizeof (*t));
2555 t->next_worker_index = next_worker_index;
2556 t->do_handoff = do_handoff;
2561 vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2564 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2566 /* Ship frames to the worker nodes */
2567 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2569 if (handoff_queue_elt_by_worker_index[i])
2571 hf = handoff_queue_elt_by_worker_index[i];
2573 * It works better to let the handoff node
2574 * rate-adapt, always ship the handoff queue element.
2576 if (1 || hf->n_vectors == hf->last_n_vectors)
2578 vlib_put_frame_queue_elt (hf);
2579 handoff_queue_elt_by_worker_index[i] = 0;
2582 hf->last_n_vectors = hf->n_vectors;
2584 congested_handoff_queue_by_worker_index[i] =
2585 (vlib_frame_queue_t *) (~0);
2588 current_worker_index = ~0;
2589 return frame->n_vectors;
2592 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2593 .function = snat_out2in_worker_handoff_fn,
2594 .name = "nat44-out2in-worker-handoff",
2595 .vector_size = sizeof (u32),
2596 .format_trace = format_snat_out2in_worker_handoff_trace,
2597 .type = VLIB_NODE_TYPE_INTERNAL,
2606 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2609 snat_out2in_fast_node_fn (vlib_main_t * vm,
2610 vlib_node_runtime_t * node,
2611 vlib_frame_t * frame)
2613 u32 n_left_from, * from, * to_next;
2614 snat_out2in_next_t next_index;
2615 u32 pkts_processed = 0;
2616 snat_main_t * sm = &snat_main;
2618 from = vlib_frame_vector_args (frame);
2619 n_left_from = frame->n_vectors;
2620 next_index = node->cached_next_index;
2622 while (n_left_from > 0)
2626 vlib_get_next_frame (vm, node, next_index,
2627 to_next, n_left_to_next);
2629 while (n_left_from > 0 && n_left_to_next > 0)
2633 u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2637 u32 new_addr0, old_addr0;
2638 u16 new_port0, old_port0;
2639 udp_header_t * udp0;
2640 tcp_header_t * tcp0;
2641 icmp46_header_t * icmp0;
2642 snat_session_key_t key0, sm0;
2646 /* speculatively enqueue b0 to the current next frame */
2652 n_left_to_next -= 1;
2654 b0 = vlib_get_buffer (vm, bi0);
2656 ip0 = vlib_buffer_get_current (b0);
2657 udp0 = ip4_next_header (ip0);
2658 tcp0 = (tcp_header_t *) udp0;
2659 icmp0 = (icmp46_header_t *) udp0;
2661 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2662 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2664 vnet_feature_next (sw_if_index0, &next0, b0);
2666 if (PREDICT_FALSE(ip0->ttl == 1))
2668 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2669 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2670 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2672 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2676 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2678 if (PREDICT_FALSE (proto0 == ~0))
2681 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2683 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2684 rx_fib_index0, node, next0, ~0, 0, 0);
2688 key0.addr = ip0->dst_address;
2689 key0.port = udp0->dst_port;
2690 key0.fib_index = rx_fib_index0;
2692 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
2694 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2698 new_addr0 = sm0.addr.as_u32;
2699 new_port0 = sm0.port;
2700 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2701 old_addr0 = ip0->dst_address.as_u32;
2702 ip0->dst_address.as_u32 = new_addr0;
2704 sum0 = ip0->checksum;
2705 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2707 dst_address /* changed member */);
2708 ip0->checksum = ip_csum_fold (sum0);
2710 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2712 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2714 old_port0 = tcp0->dst_port;
2715 tcp0->dst_port = new_port0;
2717 sum0 = tcp0->checksum;
2718 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2720 dst_address /* changed member */);
2722 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2723 ip4_header_t /* cheat */,
2724 length /* changed member */);
2725 tcp0->checksum = ip_csum_fold(sum0);
2729 old_port0 = udp0->dst_port;
2730 udp0->dst_port = new_port0;
2736 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2738 sum0 = tcp0->checksum;
2739 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2741 dst_address /* changed member */);
2743 tcp0->checksum = ip_csum_fold(sum0);
2749 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2750 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2752 snat_out2in_trace_t *t =
2753 vlib_add_trace (vm, node, b0, sizeof (*t));
2754 t->sw_if_index = sw_if_index0;
2755 t->next_index = next0;
2758 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2760 /* verify speculative enqueue, maybe switch current next frame */
2761 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2762 to_next, n_left_to_next,
2766 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2769 vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2770 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2772 return frame->n_vectors;
2775 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2776 .function = snat_out2in_fast_node_fn,
2777 .name = "nat44-out2in-fast",
2778 .vector_size = sizeof (u32),
2779 .format_trace = format_snat_out2in_fast_trace,
2780 .type = VLIB_NODE_TYPE_INTERNAL,
2782 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2783 .error_strings = snat_out2in_error_strings,
2785 .runtime_data_bytes = sizeof (snat_runtime_t),
2787 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2789 /* edit / add dispositions here */
2791 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2792 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2793 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2794 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2797 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);