2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
26 #include <nat/nat_ipfix_logging.h>
27 #include <nat/nat_det.h>
28 #include <nat/nat_reass.h>
30 #include <vppinfra/hash.h>
31 #include <vppinfra/error.h>
32 #include <vppinfra/elog.h>
38 } snat_out2in_trace_t;
41 u32 next_worker_index;
43 } snat_out2in_worker_handoff_trace_t;
45 /* packet trace format function */
46 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
48 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
49 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
50 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
52 s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
53 t->sw_if_index, t->next_index, t->session_index);
57 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
59 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
60 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
61 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
63 s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
64 t->sw_if_index, t->next_index);
68 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
70 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
71 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
72 snat_out2in_worker_handoff_trace_t * t =
73 va_arg (*args, snat_out2in_worker_handoff_trace_t *);
76 m = t->do_handoff ? "next worker" : "same worker";
77 s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
86 } nat44_out2in_reass_trace_t;
88 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
90 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
91 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
92 nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
94 s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
95 t->sw_if_index, t->next_index,
96 t->cached ? "cached" : "translated");
101 vlib_node_registration_t snat_out2in_node;
102 vlib_node_registration_t snat_out2in_fast_node;
103 vlib_node_registration_t snat_out2in_worker_handoff_node;
104 vlib_node_registration_t snat_det_out2in_node;
105 vlib_node_registration_t nat44_out2in_reass_node;
107 #define foreach_snat_out2in_error \
108 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
109 _(OUT2IN_PACKETS, "Good out2in packets processed") \
110 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
111 _(NO_TRANSLATION, "No translation") \
112 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
113 _(DROP_FRAGMENT, "Drop fragment") \
114 _(MAX_REASS, "Maximum reassemblies exceeded") \
115 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
118 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
119 foreach_snat_out2in_error
122 } snat_out2in_error_t;
124 static char * snat_out2in_error_strings[] = {
125 #define _(sym,string) string,
126 foreach_snat_out2in_error
131 SNAT_OUT2IN_NEXT_DROP,
132 SNAT_OUT2IN_NEXT_LOOKUP,
133 SNAT_OUT2IN_NEXT_ICMP_ERROR,
134 SNAT_OUT2IN_NEXT_REASS,
136 } snat_out2in_next_t;
139 * @brief Create session for static mapping.
141 * Create NAT session initiated by host from external network with static
144 * @param sm NAT main.
145 * @param b0 Vlib buffer.
146 * @param in2out In2out NAT44 session key.
147 * @param out2in Out2in NAT44 session key.
148 * @param node Vlib node.
150 * @returns SNAT session if successfully created otherwise 0.
152 static inline snat_session_t *
153 create_session_for_static_mapping (snat_main_t *sm,
155 snat_session_key_t in2out,
156 snat_session_key_t out2in,
157 vlib_node_runtime_t * node,
161 snat_user_key_t user_key;
163 clib_bihash_kv_8_8_t kv0, value0;
164 dlist_elt_t * per_user_translation_list_elt;
165 dlist_elt_t * per_user_list_head_elt;
169 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
171 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
175 ip0 = vlib_buffer_get_current (b0);
176 udp0 = ip4_next_header (ip0);
178 user_key.addr = in2out.addr;
179 user_key.fib_index = in2out.fib_index;
180 kv0.key = user_key.as_u64;
182 /* Ever heard of the "user" = inside ip4 address before? */
183 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].user_hash,
186 /* no, make a new one */
187 pool_get (sm->per_thread_data[thread_index].users, u);
188 memset (u, 0, sizeof (*u));
189 u->addr = in2out.addr;
190 u->fib_index = in2out.fib_index;
192 pool_get (sm->per_thread_data[thread_index].list_pool,
193 per_user_list_head_elt);
195 u->sessions_per_user_list_head_index = per_user_list_head_elt -
196 sm->per_thread_data[thread_index].list_pool;
198 clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
199 u->sessions_per_user_list_head_index);
201 kv0.value = u - sm->per_thread_data[thread_index].users;
204 clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].user_hash,
205 &kv0, 1 /* is_add */);
209 u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
213 pool_get (sm->per_thread_data[thread_index].sessions, s);
214 memset (s, 0, sizeof (*s));
216 s->outside_address_index = ~0;
217 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
218 s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
219 s->ext_host_port = udp0->src_port;
220 u->nstaticsessions++;
222 /* Create list elts */
223 pool_get (sm->per_thread_data[thread_index].list_pool,
224 per_user_translation_list_elt);
225 clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
226 per_user_translation_list_elt -
227 sm->per_thread_data[thread_index].list_pool);
229 per_user_translation_list_elt->value =
230 s - sm->per_thread_data[thread_index].sessions;
232 per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
233 s->per_user_list_head_index = u->sessions_per_user_list_head_index;
235 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
236 s->per_user_list_head_index,
237 per_user_translation_list_elt -
238 sm->per_thread_data[thread_index].list_pool);
242 s->in2out.protocol = out2in.protocol;
244 /* Add to translation hashes */
245 kv0.key = s->in2out.as_u64;
246 kv0.value = s - sm->per_thread_data[thread_index].sessions;
247 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
249 clib_warning ("in2out key add failed");
251 kv0.key = s->out2in.as_u64;
252 kv0.value = s - sm->per_thread_data[thread_index].sessions;
254 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
256 clib_warning ("out2in key add failed");
259 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
260 s->out2in.addr.as_u32,
264 s->in2out.fib_index);
269 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
270 snat_session_key_t *p_key0)
272 icmp46_header_t *icmp0;
273 snat_session_key_t key0;
274 icmp_echo_header_t *echo0, *inner_echo0 = 0;
275 ip4_header_t *inner_ip0;
277 icmp46_header_t *inner_icmp0;
279 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
280 echo0 = (icmp_echo_header_t *)(icmp0+1);
282 if (!icmp_is_error_message (icmp0))
284 key0.protocol = SNAT_PROTOCOL_ICMP;
285 key0.addr = ip0->dst_address;
286 key0.port = echo0->identifier;
290 inner_ip0 = (ip4_header_t *)(echo0+1);
291 l4_header = ip4_next_header (inner_ip0);
292 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
293 key0.addr = inner_ip0->src_address;
294 switch (key0.protocol)
296 case SNAT_PROTOCOL_ICMP:
297 inner_icmp0 = (icmp46_header_t*)l4_header;
298 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
299 key0.port = inner_echo0->identifier;
301 case SNAT_PROTOCOL_UDP:
302 case SNAT_PROTOCOL_TCP:
303 key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
306 return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
310 return -1; /* success */
314 * Get address and port values to be used for ICMP packet translation
315 * and create session if needed
317 * @param[in,out] sm NAT main
318 * @param[in,out] node NAT node runtime
319 * @param[in] thread_index thread index
320 * @param[in,out] b0 buffer containing packet to be translated
321 * @param[out] p_proto protocol used for matching
322 * @param[out] p_value address and port after NAT translation
323 * @param[out] p_dont_translate if packet should not be translated
324 * @param d optional parameter
325 * @param e optional parameter
327 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
328 u32 thread_index, vlib_buffer_t *b0,
329 ip4_header_t *ip0, u8 *p_proto,
330 snat_session_key_t *p_value,
331 u8 *p_dont_translate, void *d, void *e)
333 icmp46_header_t *icmp0;
336 snat_session_key_t key0;
337 snat_session_key_t sm0;
338 snat_session_t *s0 = 0;
339 u8 dont_translate = 0;
340 clib_bihash_kv_8_8_t kv0, value0;
345 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
346 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
347 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
351 err = icmp_get_key (ip0, &key0);
354 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
355 next0 = SNAT_OUT2IN_NEXT_DROP;
358 key0.fib_index = rx_fib_index0;
360 kv0.key = key0.as_u64;
362 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
365 /* Try to match static mapping by external address and port,
366 destination address and port in packet */
367 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
369 /* Don't NAT packet aimed at the intfc address */
370 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
371 ip0->dst_address.as_u32)))
376 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
377 next0 = SNAT_OUT2IN_NEXT_DROP;
381 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
382 (icmp0->type != ICMP4_echo_request || !is_addr_only)))
384 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
385 next0 = SNAT_OUT2IN_NEXT_DROP;
389 /* Create session initiated by host from external network */
390 s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
395 next0 = SNAT_OUT2IN_NEXT_DROP;
401 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
402 icmp0->type != ICMP4_echo_request &&
403 !icmp_is_error_message (icmp0)))
405 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
406 next0 = SNAT_OUT2IN_NEXT_DROP;
410 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
415 *p_proto = key0.protocol;
417 *p_value = s0->in2out;
418 *p_dont_translate = dont_translate;
420 *(snat_session_t**)d = s0;
425 * Get address and port values to be used for ICMP packet translation
427 * @param[in] sm NAT main
428 * @param[in,out] node NAT node runtime
429 * @param[in] thread_index thread index
430 * @param[in,out] b0 buffer containing packet to be translated
431 * @param[out] p_proto protocol used for matching
432 * @param[out] p_value address and port after NAT translation
433 * @param[out] p_dont_translate if packet should not be translated
434 * @param d optional parameter
435 * @param e optional parameter
437 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
438 u32 thread_index, vlib_buffer_t *b0,
439 ip4_header_t *ip0, u8 *p_proto,
440 snat_session_key_t *p_value,
441 u8 *p_dont_translate, void *d, void *e)
443 icmp46_header_t *icmp0;
446 snat_session_key_t key0;
447 snat_session_key_t sm0;
448 u8 dont_translate = 0;
453 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
454 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
455 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
457 err = icmp_get_key (ip0, &key0);
460 b0->error = node->errors[err];
461 next0 = SNAT_OUT2IN_NEXT_DROP;
464 key0.fib_index = rx_fib_index0;
466 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
468 /* Don't NAT packet aimed at the intfc address */
469 if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
474 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
475 next0 = SNAT_OUT2IN_NEXT_DROP;
479 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
480 (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
481 !icmp_is_error_message (icmp0)))
483 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
484 next0 = SNAT_OUT2IN_NEXT_DROP;
491 *p_proto = key0.protocol;
492 *p_dont_translate = dont_translate;
496 static inline u32 icmp_out2in (snat_main_t *sm,
499 icmp46_header_t * icmp0,
502 vlib_node_runtime_t * node,
508 snat_session_key_t sm0;
510 icmp_echo_header_t *echo0, *inner_echo0 = 0;
511 ip4_header_t *inner_ip0 = 0;
513 icmp46_header_t *inner_icmp0;
515 u32 new_addr0, old_addr0;
516 u16 old_id0, new_id0;
521 echo0 = (icmp_echo_header_t *)(icmp0+1);
523 next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
524 &protocol, &sm0, &dont_translate, d, e);
527 if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
530 sum0 = ip_incremental_checksum (0, icmp0,
531 ntohs(ip0->length) - ip4_header_bytes (ip0));
532 checksum0 = ~ip_csum_fold (sum0);
533 if (checksum0 != 0 && checksum0 != 0xffff)
535 next0 = SNAT_OUT2IN_NEXT_DROP;
539 old_addr0 = ip0->dst_address.as_u32;
540 new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
541 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
543 sum0 = ip0->checksum;
544 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
545 dst_address /* changed member */);
546 ip0->checksum = ip_csum_fold (sum0);
548 if (!icmp_is_error_message (icmp0))
551 if (PREDICT_FALSE(new_id0 != echo0->identifier))
553 old_id0 = echo0->identifier;
555 echo0->identifier = new_id0;
557 sum0 = icmp0->checksum;
558 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
559 identifier /* changed member */);
560 icmp0->checksum = ip_csum_fold (sum0);
565 inner_ip0 = (ip4_header_t *)(echo0+1);
566 l4_header = ip4_next_header (inner_ip0);
568 if (!ip4_header_checksum_is_valid (inner_ip0))
570 next0 = SNAT_OUT2IN_NEXT_DROP;
574 old_addr0 = inner_ip0->src_address.as_u32;
575 inner_ip0->src_address = sm0.addr;
576 new_addr0 = inner_ip0->src_address.as_u32;
578 sum0 = icmp0->checksum;
579 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
580 src_address /* changed member */);
581 icmp0->checksum = ip_csum_fold (sum0);
585 case SNAT_PROTOCOL_ICMP:
586 inner_icmp0 = (icmp46_header_t*)l4_header;
587 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
589 old_id0 = inner_echo0->identifier;
591 inner_echo0->identifier = new_id0;
593 sum0 = icmp0->checksum;
594 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
596 icmp0->checksum = ip_csum_fold (sum0);
598 case SNAT_PROTOCOL_UDP:
599 case SNAT_PROTOCOL_TCP:
600 old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
602 ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
604 sum0 = icmp0->checksum;
605 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
607 icmp0->checksum = ip_csum_fold (sum0);
619 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
622 icmp46_header_t * icmp0,
625 vlib_node_runtime_t * node,
628 snat_session_t ** p_s0)
630 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
631 next0, thread_index, p_s0, 0);
632 snat_session_t * s0 = *p_s0;
633 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
636 s0->last_heard = now;
638 s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
639 /* Per-user LRU list maintenance for dynamic translation */
640 if (!snat_is_session_static (s0))
642 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
644 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
645 s0->per_user_list_head_index,
652 static snat_session_t *
653 snat_out2in_unknown_proto (snat_main_t *sm,
660 vlib_node_runtime_t * node)
662 clib_bihash_kv_8_8_t kv, value;
663 clib_bihash_kv_16_8_t s_kv, s_value;
664 snat_static_mapping_t *m;
665 snat_session_key_t m_key;
666 u32 old_addr, new_addr;
668 nat_ed_ses_key_t key;
670 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
671 snat_user_key_t u_key;
673 dlist_elt_t *head, *elt;
675 old_addr = ip->dst_address.as_u32;
677 key.l_addr = ip->dst_address;
678 key.r_addr = ip->src_address;
679 key.fib_index = rx_fib_index;
680 key.proto = ip->protocol;
683 s_kv.key[0] = key.as_u64[0];
684 s_kv.key[1] = key.as_u64[1];
686 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
688 s = pool_elt_at_index (tsm->sessions, s_value.value);
689 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
693 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
695 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
699 m_key.addr = ip->dst_address;
702 m_key.fib_index = rx_fib_index;
703 kv.key = m_key.as_u64;
704 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
706 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
710 m = pool_elt_at_index (sm->static_mappings, value.value);
712 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
714 u_key.addr = ip->src_address;
715 u_key.fib_index = m->fib_index;
716 kv.key = u_key.as_u64;
718 /* Ever heard of the "user" = src ip4 address before? */
719 if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
721 /* no, make a new one */
722 pool_get (tsm->users, u);
723 memset (u, 0, sizeof (*u));
724 u->addr = ip->src_address;
725 u->fib_index = rx_fib_index;
727 pool_get (tsm->list_pool, head);
728 u->sessions_per_user_list_head_index = head - tsm->list_pool;
730 clib_dlist_init (tsm->list_pool,
731 u->sessions_per_user_list_head_index);
733 kv.value = u - tsm->users;
736 clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1);
740 u = pool_elt_at_index (tsm->users, value.value);
743 /* Create a new session */
744 pool_get (tsm->sessions, s);
745 memset (s, 0, sizeof (*s));
747 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
748 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
749 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
750 s->outside_address_index = ~0;
751 s->out2in.addr.as_u32 = old_addr;
752 s->out2in.fib_index = rx_fib_index;
753 s->in2out.addr.as_u32 = new_addr;
754 s->in2out.fib_index = m->fib_index;
755 s->in2out.port = s->out2in.port = ip->protocol;
756 u->nstaticsessions++;
758 /* Create list elts */
759 pool_get (tsm->list_pool, elt);
760 clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
761 elt->value = s - tsm->sessions;
762 s->per_user_index = elt - tsm->list_pool;
763 s->per_user_list_head_index = u->sessions_per_user_list_head_index;
764 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
767 /* Add to lookup tables */
768 s_kv.value = s - tsm->sessions;
769 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
770 clib_warning ("out2in key add failed");
772 key.l_addr = ip->dst_address;
773 key.fib_index = m->fib_index;
774 s_kv.key[0] = key.as_u64[0];
775 s_kv.key[1] = key.as_u64[1];
776 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
777 clib_warning ("in2out key add failed");
780 /* Update IP checksum */
782 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
783 ip->checksum = ip_csum_fold (sum);
785 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
790 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
791 /* Per-user LRU list maintenance */
792 clib_dlist_remove (tsm->list_pool, s->per_user_index);
793 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
799 static snat_session_t *
800 snat_out2in_lb (snat_main_t *sm,
807 vlib_node_runtime_t * node)
809 nat_ed_ses_key_t key;
810 clib_bihash_kv_16_8_t s_kv, s_value;
811 udp_header_t *udp = ip4_next_header (ip);
812 tcp_header_t *tcp = (tcp_header_t *) udp;
813 snat_session_t *s = 0;
814 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
815 snat_session_key_t e_key, l_key;
816 clib_bihash_kv_8_8_t kv, value;
817 u32 old_addr, new_addr;
818 u32 proto = ip_proto_to_snat_proto (ip->protocol);
819 u16 new_port, old_port;
821 snat_user_key_t u_key;
823 dlist_elt_t *head, *elt;
825 old_addr = ip->dst_address.as_u32;
827 key.l_addr = ip->dst_address;
828 key.r_addr = ip->src_address;
829 key.fib_index = rx_fib_index;
830 key.proto = ip->protocol;
832 key.l_port = udp->dst_port;
833 s_kv.key[0] = key.as_u64[0];
834 s_kv.key[1] = key.as_u64[1];
836 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
838 s = pool_elt_at_index (tsm->sessions, s_value.value);
842 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
844 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
848 e_key.addr = ip->dst_address;
849 e_key.port = udp->dst_port;
850 e_key.protocol = proto;
851 e_key.fib_index = rx_fib_index;
852 if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0))
855 u_key.addr = l_key.addr;
856 u_key.fib_index = l_key.fib_index;
857 kv.key = u_key.as_u64;
859 /* Ever heard of the "user" = src ip4 address before? */
860 if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
862 /* no, make a new one */
863 pool_get (tsm->users, u);
864 memset (u, 0, sizeof (*u));
865 u->addr = l_key.addr;
866 u->fib_index = l_key.fib_index;
868 pool_get (tsm->list_pool, head);
869 u->sessions_per_user_list_head_index = head - tsm->list_pool;
871 clib_dlist_init (tsm->list_pool,
872 u->sessions_per_user_list_head_index);
874 kv.value = u - tsm->users;
877 if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
878 clib_warning ("user key add failed");
882 u = pool_elt_at_index (tsm->users, value.value);
885 /* Create a new session */
886 pool_get (tsm->sessions, s);
887 memset (s, 0, sizeof (*s));
889 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
890 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
891 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
892 s->outside_address_index = ~0;
895 u->nstaticsessions++;
897 /* Create list elts */
898 pool_get (tsm->list_pool, elt);
899 clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
900 elt->value = s - tsm->sessions;
901 s->per_user_index = elt - tsm->list_pool;
902 s->per_user_list_head_index = u->sessions_per_user_list_head_index;
903 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
906 /* Add to lookup tables */
907 s_kv.value = s - tsm->sessions;
908 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
909 clib_warning ("out2in-ed key add failed");
911 key.l_addr = l_key.addr;
912 key.fib_index = l_key.fib_index;
913 key.l_port = l_key.port;
914 s_kv.key[0] = key.as_u64[0];
915 s_kv.key[1] = key.as_u64[1];
916 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
917 clib_warning ("in2out-ed key add failed");
920 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
922 /* Update IP checksum */
924 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
925 ip->checksum = ip_csum_fold (sum);
927 if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
929 old_port = tcp->dst_port;
930 tcp->dst_port = s->in2out.port;
931 new_port = tcp->dst_port;
934 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
935 sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
936 tcp->checksum = ip_csum_fold(sum);
940 udp->dst_port = s->in2out.port;
944 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
949 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
954 snat_out2in_node_fn (vlib_main_t * vm,
955 vlib_node_runtime_t * node,
956 vlib_frame_t * frame)
958 u32 n_left_from, * from, * to_next;
959 snat_out2in_next_t next_index;
960 u32 pkts_processed = 0;
961 snat_main_t * sm = &snat_main;
962 f64 now = vlib_time_now (vm);
963 u32 thread_index = vlib_get_thread_index ();
965 from = vlib_frame_vector_args (frame);
966 n_left_from = frame->n_vectors;
967 next_index = node->cached_next_index;
969 while (n_left_from > 0)
973 vlib_get_next_frame (vm, node, next_index,
974 to_next, n_left_to_next);
976 while (n_left_from >= 4 && n_left_to_next >= 2)
979 vlib_buffer_t * b0, * b1;
980 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
981 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
982 u32 sw_if_index0, sw_if_index1;
983 ip4_header_t * ip0, *ip1;
984 ip_csum_t sum0, sum1;
985 u32 new_addr0, old_addr0;
986 u16 new_port0, old_port0;
987 u32 new_addr1, old_addr1;
988 u16 new_port1, old_port1;
989 udp_header_t * udp0, * udp1;
990 tcp_header_t * tcp0, * tcp1;
991 icmp46_header_t * icmp0, * icmp1;
992 snat_session_key_t key0, key1, sm0, sm1;
993 u32 rx_fib_index0, rx_fib_index1;
995 snat_session_t * s0 = 0, * s1 = 0;
996 clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
998 /* Prefetch next iteration. */
1000 vlib_buffer_t * p2, * p3;
1002 p2 = vlib_get_buffer (vm, from[2]);
1003 p3 = vlib_get_buffer (vm, from[3]);
1005 vlib_prefetch_buffer_header (p2, LOAD);
1006 vlib_prefetch_buffer_header (p3, LOAD);
1008 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1009 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1012 /* speculatively enqueue b0 and b1 to the current next frame */
1013 to_next[0] = bi0 = from[0];
1014 to_next[1] = bi1 = from[1];
1018 n_left_to_next -= 2;
1020 b0 = vlib_get_buffer (vm, bi0);
1021 b1 = vlib_get_buffer (vm, bi1);
1023 vnet_buffer (b0)->snat.flags = 0;
1024 vnet_buffer (b1)->snat.flags = 0;
1026 ip0 = vlib_buffer_get_current (b0);
1027 udp0 = ip4_next_header (ip0);
1028 tcp0 = (tcp_header_t *) udp0;
1029 icmp0 = (icmp46_header_t *) udp0;
1031 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1032 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1035 if (PREDICT_FALSE(ip0->ttl == 1))
1037 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1038 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1039 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1041 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1045 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1047 if (PREDICT_FALSE (proto0 == ~0))
1049 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1050 thread_index, now, vm, node);
1052 next0 = SNAT_OUT2IN_NEXT_DROP;
1056 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1058 next0 = icmp_out2in_slow_path
1059 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1060 next0, now, thread_index, &s0);
1064 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1066 next0 = SNAT_OUT2IN_NEXT_REASS;
1070 key0.addr = ip0->dst_address;
1071 key0.port = udp0->dst_port;
1072 key0.protocol = proto0;
1073 key0.fib_index = rx_fib_index0;
1075 kv0.key = key0.as_u64;
1077 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1080 /* Try to match static mapping by external address and port,
1081 destination address and port in packet */
1082 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1084 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1086 * Send DHCP packets to the ipv4 stack, or we won't
1087 * be able to use dhcp client on the outside interface
1089 if (proto0 != SNAT_PROTOCOL_UDP
1091 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1092 next0 = SNAT_OUT2IN_NEXT_DROP;
1096 /* Create session initiated by host from external network */
1097 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1101 next0 = SNAT_OUT2IN_NEXT_DROP;
1107 if (PREDICT_FALSE (value0.value == ~0ULL))
1109 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1112 next0 = SNAT_OUT2IN_NEXT_DROP;
1117 s0 = pool_elt_at_index (
1118 sm->per_thread_data[thread_index].sessions,
1123 old_addr0 = ip0->dst_address.as_u32;
1124 ip0->dst_address = s0->in2out.addr;
1125 new_addr0 = ip0->dst_address.as_u32;
1126 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1128 sum0 = ip0->checksum;
1129 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1131 dst_address /* changed member */);
1132 ip0->checksum = ip_csum_fold (sum0);
1134 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1136 old_port0 = tcp0->dst_port;
1137 tcp0->dst_port = s0->in2out.port;
1138 new_port0 = tcp0->dst_port;
1140 sum0 = tcp0->checksum;
1141 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1143 dst_address /* changed member */);
1145 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1146 ip4_header_t /* cheat */,
1147 length /* changed member */);
1148 tcp0->checksum = ip_csum_fold(sum0);
1152 old_port0 = udp0->dst_port;
1153 udp0->dst_port = s0->in2out.port;
1158 s0->last_heard = now;
1160 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1161 /* Per-user LRU list maintenance for dynamic translation */
1162 if (!snat_is_session_static (s0))
1164 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1165 s0->per_user_index);
1166 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1167 s0->per_user_list_head_index,
1168 s0->per_user_index);
1172 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1173 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1175 snat_out2in_trace_t *t =
1176 vlib_add_trace (vm, node, b0, sizeof (*t));
1177 t->sw_if_index = sw_if_index0;
1178 t->next_index = next0;
1179 t->session_index = ~0;
1181 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1184 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1187 ip1 = vlib_buffer_get_current (b1);
1188 udp1 = ip4_next_header (ip1);
1189 tcp1 = (tcp_header_t *) udp1;
1190 icmp1 = (icmp46_header_t *) udp1;
1192 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1193 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1196 if (PREDICT_FALSE(ip1->ttl == 1))
1198 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1199 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1200 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1202 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1206 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1208 if (PREDICT_FALSE (proto1 == ~0))
1210 s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1211 thread_index, now, vm, node);
1213 next1 = SNAT_OUT2IN_NEXT_DROP;
1217 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1219 next1 = icmp_out2in_slow_path
1220 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1221 next1, now, thread_index, &s1);
1225 if (PREDICT_FALSE (ip4_is_fragment (ip1)))
1227 next1 = SNAT_OUT2IN_NEXT_REASS;
1231 key1.addr = ip1->dst_address;
1232 key1.port = udp1->dst_port;
1233 key1.protocol = proto1;
1234 key1.fib_index = rx_fib_index1;
1236 kv1.key = key1.as_u64;
1238 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1241 /* Try to match static mapping by external address and port,
1242 destination address and port in packet */
1243 if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
1245 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1247 * Send DHCP packets to the ipv4 stack, or we won't
1248 * be able to use dhcp client on the outside interface
1250 if (proto1 != SNAT_PROTOCOL_UDP
1252 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1253 next1 = SNAT_OUT2IN_NEXT_DROP;
1257 /* Create session initiated by host from external network */
1258 s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1262 next1 = SNAT_OUT2IN_NEXT_DROP;
1268 if (PREDICT_FALSE (value1.value == ~0ULL))
1270 s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1273 next1 = SNAT_OUT2IN_NEXT_DROP;
1278 s1 = pool_elt_at_index (
1279 sm->per_thread_data[thread_index].sessions,
1284 old_addr1 = ip1->dst_address.as_u32;
1285 ip1->dst_address = s1->in2out.addr;
1286 new_addr1 = ip1->dst_address.as_u32;
1287 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1289 sum1 = ip1->checksum;
1290 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1292 dst_address /* changed member */);
1293 ip1->checksum = ip_csum_fold (sum1);
1295 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1297 old_port1 = tcp1->dst_port;
1298 tcp1->dst_port = s1->in2out.port;
1299 new_port1 = tcp1->dst_port;
1301 sum1 = tcp1->checksum;
1302 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1304 dst_address /* changed member */);
1306 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1307 ip4_header_t /* cheat */,
1308 length /* changed member */);
1309 tcp1->checksum = ip_csum_fold(sum1);
1313 old_port1 = udp1->dst_port;
1314 udp1->dst_port = s1->in2out.port;
1319 s1->last_heard = now;
1321 s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1322 /* Per-user LRU list maintenance for dynamic translation */
1323 if (!snat_is_session_static (s1))
1325 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1326 s1->per_user_index);
1327 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1328 s1->per_user_list_head_index,
1329 s1->per_user_index);
1333 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1334 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1336 snat_out2in_trace_t *t =
1337 vlib_add_trace (vm, node, b1, sizeof (*t));
1338 t->sw_if_index = sw_if_index1;
1339 t->next_index = next1;
1340 t->session_index = ~0;
1342 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1345 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1347 /* verify speculative enqueues, maybe switch current next frame */
1348 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1349 to_next, n_left_to_next,
1350 bi0, bi1, next0, next1);
1353 while (n_left_from > 0 && n_left_to_next > 0)
1357 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1361 u32 new_addr0, old_addr0;
1362 u16 new_port0, old_port0;
1363 udp_header_t * udp0;
1364 tcp_header_t * tcp0;
1365 icmp46_header_t * icmp0;
1366 snat_session_key_t key0, sm0;
1369 snat_session_t * s0 = 0;
1370 clib_bihash_kv_8_8_t kv0, value0;
1372 /* speculatively enqueue b0 to the current next frame */
1378 n_left_to_next -= 1;
1380 b0 = vlib_get_buffer (vm, bi0);
1382 vnet_buffer (b0)->snat.flags = 0;
1384 ip0 = vlib_buffer_get_current (b0);
1385 udp0 = ip4_next_header (ip0);
1386 tcp0 = (tcp_header_t *) udp0;
1387 icmp0 = (icmp46_header_t *) udp0;
1389 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1390 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1393 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1395 if (PREDICT_FALSE (proto0 == ~0))
1397 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1398 thread_index, now, vm, node);
1400 next0 = SNAT_OUT2IN_NEXT_DROP;
1404 if (PREDICT_FALSE(ip0->ttl == 1))
1406 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1407 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1408 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1410 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1414 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1416 next0 = icmp_out2in_slow_path
1417 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1418 next0, now, thread_index, &s0);
1422 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1424 next0 = SNAT_OUT2IN_NEXT_REASS;
1428 key0.addr = ip0->dst_address;
1429 key0.port = udp0->dst_port;
1430 key0.protocol = proto0;
1431 key0.fib_index = rx_fib_index0;
1433 kv0.key = key0.as_u64;
1435 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1438 /* Try to match static mapping by external address and port,
1439 destination address and port in packet */
1440 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1442 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1444 * Send DHCP packets to the ipv4 stack, or we won't
1445 * be able to use dhcp client on the outside interface
1447 if (proto0 != SNAT_PROTOCOL_UDP
1449 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1451 next0 = SNAT_OUT2IN_NEXT_DROP;
1455 /* Create session initiated by host from external network */
1456 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1460 next0 = SNAT_OUT2IN_NEXT_DROP;
1466 if (PREDICT_FALSE (value0.value == ~0ULL))
1468 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1471 next0 = SNAT_OUT2IN_NEXT_DROP;
1476 s0 = pool_elt_at_index (
1477 sm->per_thread_data[thread_index].sessions,
1482 old_addr0 = ip0->dst_address.as_u32;
1483 ip0->dst_address = s0->in2out.addr;
1484 new_addr0 = ip0->dst_address.as_u32;
1485 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1487 sum0 = ip0->checksum;
1488 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1490 dst_address /* changed member */);
1491 ip0->checksum = ip_csum_fold (sum0);
1493 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1495 old_port0 = tcp0->dst_port;
1496 tcp0->dst_port = s0->in2out.port;
1497 new_port0 = tcp0->dst_port;
1499 sum0 = tcp0->checksum;
1500 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1502 dst_address /* changed member */);
1504 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1505 ip4_header_t /* cheat */,
1506 length /* changed member */);
1507 tcp0->checksum = ip_csum_fold(sum0);
1511 old_port0 = udp0->dst_port;
1512 udp0->dst_port = s0->in2out.port;
1517 s0->last_heard = now;
1519 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1520 /* Per-user LRU list maintenance for dynamic translation */
1521 if (!snat_is_session_static (s0))
1523 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1524 s0->per_user_index);
1525 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1526 s0->per_user_list_head_index,
1527 s0->per_user_index);
1531 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1532 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1534 snat_out2in_trace_t *t =
1535 vlib_add_trace (vm, node, b0, sizeof (*t));
1536 t->sw_if_index = sw_if_index0;
1537 t->next_index = next0;
1538 t->session_index = ~0;
1540 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1543 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1545 /* verify speculative enqueue, maybe switch current next frame */
1546 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1547 to_next, n_left_to_next,
1551 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1554 vlib_node_increment_counter (vm, snat_out2in_node.index,
1555 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1557 return frame->n_vectors;
1560 VLIB_REGISTER_NODE (snat_out2in_node) = {
1561 .function = snat_out2in_node_fn,
1562 .name = "nat44-out2in",
1563 .vector_size = sizeof (u32),
1564 .format_trace = format_snat_out2in_trace,
1565 .type = VLIB_NODE_TYPE_INTERNAL,
1567 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1568 .error_strings = snat_out2in_error_strings,
1570 .runtime_data_bytes = sizeof (snat_runtime_t),
1572 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1574 /* edit / add dispositions here */
1576 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1577 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1578 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1579 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1582 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1585 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1586 vlib_node_runtime_t * node,
1587 vlib_frame_t * frame)
1589 u32 n_left_from, *from, *to_next;
1590 snat_out2in_next_t next_index;
1591 u32 pkts_processed = 0;
1592 snat_main_t *sm = &snat_main;
1593 f64 now = vlib_time_now (vm);
1594 u32 thread_index = vlib_get_thread_index ();
1595 snat_main_per_thread_data_t *per_thread_data =
1596 &sm->per_thread_data[thread_index];
1597 u32 *fragments_to_drop = 0;
1598 u32 *fragments_to_loopback = 0;
1600 from = vlib_frame_vector_args (frame);
1601 n_left_from = frame->n_vectors;
1602 next_index = node->cached_next_index;
1604 while (n_left_from > 0)
1608 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1610 while (n_left_from > 0 && n_left_to_next > 0)
1612 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1617 nat_reass_ip4_t *reass0;
1618 udp_header_t * udp0;
1619 tcp_header_t * tcp0;
1620 snat_session_key_t key0, sm0;
1621 clib_bihash_kv_8_8_t kv0, value0;
1622 snat_session_t * s0 = 0;
1623 u16 old_port0, new_port0;
1626 /* speculatively enqueue b0 to the current next frame */
1632 n_left_to_next -= 1;
1634 b0 = vlib_get_buffer (vm, bi0);
1635 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1637 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1638 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1641 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1643 next0 = SNAT_OUT2IN_NEXT_DROP;
1644 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1648 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1649 udp0 = ip4_next_header (ip0);
1650 tcp0 = (tcp_header_t *) udp0;
1651 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1653 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1658 &fragments_to_drop);
1660 if (PREDICT_FALSE (!reass0))
1662 next0 = SNAT_OUT2IN_NEXT_DROP;
1663 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1667 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1669 key0.addr = ip0->dst_address;
1670 key0.port = udp0->dst_port;
1671 key0.protocol = proto0;
1672 key0.fib_index = rx_fib_index0;
1673 kv0.key = key0.as_u64;
1675 if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1677 /* Try to match static mapping by external address and port,
1678 destination address and port in packet */
1679 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1681 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1683 * Send DHCP packets to the ipv4 stack, or we won't
1684 * be able to use dhcp client on the outside interface
1686 if (proto0 != SNAT_PROTOCOL_UDP
1688 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1690 next0 = SNAT_OUT2IN_NEXT_DROP;
1694 /* Create session initiated by host from external network */
1695 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1699 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1700 next0 = SNAT_OUT2IN_NEXT_DROP;
1703 reass0->sess_index = s0 - per_thread_data->sessions;
1707 s0 = pool_elt_at_index (per_thread_data->sessions,
1709 reass0->sess_index = value0.value;
1711 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1715 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1717 if (nat_ip4_reass_add_fragment (reass0, bi0))
1719 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1720 next0 = SNAT_OUT2IN_NEXT_DROP;
1726 s0 = pool_elt_at_index (per_thread_data->sessions,
1727 reass0->sess_index);
1730 old_addr0 = ip0->dst_address.as_u32;
1731 ip0->dst_address = s0->in2out.addr;
1732 new_addr0 = ip0->dst_address.as_u32;
1733 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1735 sum0 = ip0->checksum;
1736 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1738 dst_address /* changed member */);
1739 ip0->checksum = ip_csum_fold (sum0);
1741 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1743 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1745 old_port0 = tcp0->dst_port;
1746 tcp0->dst_port = s0->in2out.port;
1747 new_port0 = tcp0->dst_port;
1749 sum0 = tcp0->checksum;
1750 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1752 dst_address /* changed member */);
1754 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1755 ip4_header_t /* cheat */,
1756 length /* changed member */);
1757 tcp0->checksum = ip_csum_fold(sum0);
1761 old_port0 = udp0->dst_port;
1762 udp0->dst_port = s0->in2out.port;
1768 s0->last_heard = now;
1770 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1771 /* Per-user LRU list maintenance for dynamic translation */
1772 if (!snat_is_session_static (s0))
1774 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1775 s0->per_user_index);
1776 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1777 s0->per_user_list_head_index,
1778 s0->per_user_index);
1782 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1783 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1785 nat44_out2in_reass_trace_t *t =
1786 vlib_add_trace (vm, node, b0, sizeof (*t));
1787 t->cached = cached0;
1788 t->sw_if_index = sw_if_index0;
1789 t->next_index = next0;
1799 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1801 /* verify speculative enqueue, maybe switch current next frame */
1802 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1803 to_next, n_left_to_next,
1807 if (n_left_from == 0 && vec_len (fragments_to_loopback))
1809 from = vlib_frame_vector_args (frame);
1810 u32 len = vec_len (fragments_to_loopback);
1811 if (len <= VLIB_FRAME_SIZE)
1813 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1815 vec_reset_length (fragments_to_loopback);
1820 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1821 sizeof (u32) * VLIB_FRAME_SIZE);
1822 n_left_from = VLIB_FRAME_SIZE;
1823 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1828 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1831 vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1832 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1835 nat_send_all_to_node (vm, fragments_to_drop, node,
1836 &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1837 SNAT_OUT2IN_NEXT_DROP);
1839 vec_free (fragments_to_drop);
1840 vec_free (fragments_to_loopback);
1841 return frame->n_vectors;
1844 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1845 .function = nat44_out2in_reass_node_fn,
1846 .name = "nat44-out2in-reass",
1847 .vector_size = sizeof (u32),
1848 .format_trace = format_nat44_out2in_reass_trace,
1849 .type = VLIB_NODE_TYPE_INTERNAL,
1851 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1852 .error_strings = snat_out2in_error_strings,
1854 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1856 /* edit / add dispositions here */
1858 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1859 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1860 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1861 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1864 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1865 nat44_out2in_reass_node_fn);
1867 /**************************/
1868 /*** deterministic mode ***/
1869 /**************************/
1871 snat_det_out2in_node_fn (vlib_main_t * vm,
1872 vlib_node_runtime_t * node,
1873 vlib_frame_t * frame)
1875 u32 n_left_from, * from, * to_next;
1876 snat_out2in_next_t next_index;
1877 u32 pkts_processed = 0;
1878 snat_main_t * sm = &snat_main;
1879 u32 thread_index = vlib_get_thread_index ();
1881 from = vlib_frame_vector_args (frame);
1882 n_left_from = frame->n_vectors;
1883 next_index = node->cached_next_index;
1885 while (n_left_from > 0)
1889 vlib_get_next_frame (vm, node, next_index,
1890 to_next, n_left_to_next);
1892 while (n_left_from >= 4 && n_left_to_next >= 2)
1895 vlib_buffer_t * b0, * b1;
1896 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1897 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1898 u32 sw_if_index0, sw_if_index1;
1899 ip4_header_t * ip0, * ip1;
1900 ip_csum_t sum0, sum1;
1901 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1902 u16 new_port0, old_port0, old_port1, new_port1;
1903 udp_header_t * udp0, * udp1;
1904 tcp_header_t * tcp0, * tcp1;
1906 snat_det_out_key_t key0, key1;
1907 snat_det_map_t * dm0, * dm1;
1908 snat_det_session_t * ses0 = 0, * ses1 = 0;
1909 u32 rx_fib_index0, rx_fib_index1;
1910 icmp46_header_t * icmp0, * icmp1;
1912 /* Prefetch next iteration. */
1914 vlib_buffer_t * p2, * p3;
1916 p2 = vlib_get_buffer (vm, from[2]);
1917 p3 = vlib_get_buffer (vm, from[3]);
1919 vlib_prefetch_buffer_header (p2, LOAD);
1920 vlib_prefetch_buffer_header (p3, LOAD);
1922 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1923 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1926 /* speculatively enqueue b0 and b1 to the current next frame */
1927 to_next[0] = bi0 = from[0];
1928 to_next[1] = bi1 = from[1];
1932 n_left_to_next -= 2;
1934 b0 = vlib_get_buffer (vm, bi0);
1935 b1 = vlib_get_buffer (vm, bi1);
1937 ip0 = vlib_buffer_get_current (b0);
1938 udp0 = ip4_next_header (ip0);
1939 tcp0 = (tcp_header_t *) udp0;
1941 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1943 if (PREDICT_FALSE(ip0->ttl == 1))
1945 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1946 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1947 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1949 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1953 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1955 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1957 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1958 icmp0 = (icmp46_header_t *) udp0;
1960 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1961 rx_fib_index0, node, next0, thread_index,
1966 key0.ext_host_addr = ip0->src_address;
1967 key0.ext_host_port = tcp0->src;
1968 key0.out_port = tcp0->dst;
1970 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1971 if (PREDICT_FALSE(!dm0))
1973 clib_warning("unknown dst address: %U",
1974 format_ip4_address, &ip0->dst_address);
1975 next0 = SNAT_OUT2IN_NEXT_DROP;
1976 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1980 snat_det_reverse(dm0, &ip0->dst_address,
1981 clib_net_to_host_u16(tcp0->dst), &new_addr0);
1983 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1984 if (PREDICT_FALSE(!ses0))
1986 clib_warning("no match src %U:%d dst %U:%d for user %U",
1987 format_ip4_address, &ip0->src_address,
1988 clib_net_to_host_u16 (tcp0->src),
1989 format_ip4_address, &ip0->dst_address,
1990 clib_net_to_host_u16 (tcp0->dst),
1991 format_ip4_address, &new_addr0);
1992 next0 = SNAT_OUT2IN_NEXT_DROP;
1993 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1996 new_port0 = ses0->in_port;
1998 old_addr0 = ip0->dst_address;
1999 ip0->dst_address = new_addr0;
2000 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2002 sum0 = ip0->checksum;
2003 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2005 dst_address /* changed member */);
2006 ip0->checksum = ip_csum_fold (sum0);
2008 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2010 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2011 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2012 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2013 snat_det_ses_close(dm0, ses0);
2015 old_port0 = tcp0->dst;
2016 tcp0->dst = new_port0;
2018 sum0 = tcp0->checksum;
2019 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2021 dst_address /* changed member */);
2023 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2024 ip4_header_t /* cheat */,
2025 length /* changed member */);
2026 tcp0->checksum = ip_csum_fold(sum0);
2030 old_port0 = udp0->dst_port;
2031 udp0->dst_port = new_port0;
2037 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2038 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2040 snat_out2in_trace_t *t =
2041 vlib_add_trace (vm, node, b0, sizeof (*t));
2042 t->sw_if_index = sw_if_index0;
2043 t->next_index = next0;
2044 t->session_index = ~0;
2046 t->session_index = ses0 - dm0->sessions;
2049 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2051 b1 = vlib_get_buffer (vm, bi1);
2053 ip1 = vlib_buffer_get_current (b1);
2054 udp1 = ip4_next_header (ip1);
2055 tcp1 = (tcp_header_t *) udp1;
2057 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2059 if (PREDICT_FALSE(ip1->ttl == 1))
2061 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2062 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2063 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2065 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2069 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2071 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2073 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2074 icmp1 = (icmp46_header_t *) udp1;
2076 next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
2077 rx_fib_index1, node, next1, thread_index,
2082 key1.ext_host_addr = ip1->src_address;
2083 key1.ext_host_port = tcp1->src;
2084 key1.out_port = tcp1->dst;
2086 dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
2087 if (PREDICT_FALSE(!dm1))
2089 clib_warning("unknown dst address: %U",
2090 format_ip4_address, &ip1->dst_address);
2091 next1 = SNAT_OUT2IN_NEXT_DROP;
2092 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2096 snat_det_reverse(dm1, &ip1->dst_address,
2097 clib_net_to_host_u16(tcp1->dst), &new_addr1);
2099 ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
2100 if (PREDICT_FALSE(!ses1))
2102 clib_warning("no match src %U:%d dst %U:%d for user %U",
2103 format_ip4_address, &ip1->src_address,
2104 clib_net_to_host_u16 (tcp1->src),
2105 format_ip4_address, &ip1->dst_address,
2106 clib_net_to_host_u16 (tcp1->dst),
2107 format_ip4_address, &new_addr1);
2108 next1 = SNAT_OUT2IN_NEXT_DROP;
2109 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2112 new_port1 = ses1->in_port;
2114 old_addr1 = ip1->dst_address;
2115 ip1->dst_address = new_addr1;
2116 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2118 sum1 = ip1->checksum;
2119 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2121 dst_address /* changed member */);
2122 ip1->checksum = ip_csum_fold (sum1);
2124 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
2126 if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
2127 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2128 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
2129 snat_det_ses_close(dm1, ses1);
2131 old_port1 = tcp1->dst;
2132 tcp1->dst = new_port1;
2134 sum1 = tcp1->checksum;
2135 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
2137 dst_address /* changed member */);
2139 sum1 = ip_csum_update (sum1, old_port1, new_port1,
2140 ip4_header_t /* cheat */,
2141 length /* changed member */);
2142 tcp1->checksum = ip_csum_fold(sum1);
2146 old_port1 = udp1->dst_port;
2147 udp1->dst_port = new_port1;
2153 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2154 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2156 snat_out2in_trace_t *t =
2157 vlib_add_trace (vm, node, b1, sizeof (*t));
2158 t->sw_if_index = sw_if_index1;
2159 t->next_index = next1;
2160 t->session_index = ~0;
2162 t->session_index = ses1 - dm1->sessions;
2165 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
2167 /* verify speculative enqueues, maybe switch current next frame */
2168 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2169 to_next, n_left_to_next,
2170 bi0, bi1, next0, next1);
2173 while (n_left_from > 0 && n_left_to_next > 0)
2177 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2181 ip4_address_t new_addr0, old_addr0;
2182 u16 new_port0, old_port0;
2183 udp_header_t * udp0;
2184 tcp_header_t * tcp0;
2186 snat_det_out_key_t key0;
2187 snat_det_map_t * dm0;
2188 snat_det_session_t * ses0 = 0;
2190 icmp46_header_t * icmp0;
2192 /* speculatively enqueue b0 to the current next frame */
2198 n_left_to_next -= 1;
2200 b0 = vlib_get_buffer (vm, bi0);
2202 ip0 = vlib_buffer_get_current (b0);
2203 udp0 = ip4_next_header (ip0);
2204 tcp0 = (tcp_header_t *) udp0;
2206 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2208 if (PREDICT_FALSE(ip0->ttl == 1))
2210 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2211 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2212 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2214 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2218 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2220 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2222 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2223 icmp0 = (icmp46_header_t *) udp0;
2225 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2226 rx_fib_index0, node, next0, thread_index,
2231 key0.ext_host_addr = ip0->src_address;
2232 key0.ext_host_port = tcp0->src;
2233 key0.out_port = tcp0->dst;
2235 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2236 if (PREDICT_FALSE(!dm0))
2238 clib_warning("unknown dst address: %U",
2239 format_ip4_address, &ip0->dst_address);
2240 next0 = SNAT_OUT2IN_NEXT_DROP;
2241 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2245 snat_det_reverse(dm0, &ip0->dst_address,
2246 clib_net_to_host_u16(tcp0->dst), &new_addr0);
2248 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2249 if (PREDICT_FALSE(!ses0))
2251 clib_warning("no match src %U:%d dst %U:%d for user %U",
2252 format_ip4_address, &ip0->src_address,
2253 clib_net_to_host_u16 (tcp0->src),
2254 format_ip4_address, &ip0->dst_address,
2255 clib_net_to_host_u16 (tcp0->dst),
2256 format_ip4_address, &new_addr0);
2257 next0 = SNAT_OUT2IN_NEXT_DROP;
2258 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2261 new_port0 = ses0->in_port;
2263 old_addr0 = ip0->dst_address;
2264 ip0->dst_address = new_addr0;
2265 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2267 sum0 = ip0->checksum;
2268 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2270 dst_address /* changed member */);
2271 ip0->checksum = ip_csum_fold (sum0);
2273 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2275 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2276 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2277 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2278 snat_det_ses_close(dm0, ses0);
2280 old_port0 = tcp0->dst;
2281 tcp0->dst = new_port0;
2283 sum0 = tcp0->checksum;
2284 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2286 dst_address /* changed member */);
2288 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2289 ip4_header_t /* cheat */,
2290 length /* changed member */);
2291 tcp0->checksum = ip_csum_fold(sum0);
2295 old_port0 = udp0->dst_port;
2296 udp0->dst_port = new_port0;
2302 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2303 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2305 snat_out2in_trace_t *t =
2306 vlib_add_trace (vm, node, b0, sizeof (*t));
2307 t->sw_if_index = sw_if_index0;
2308 t->next_index = next0;
2309 t->session_index = ~0;
2311 t->session_index = ses0 - dm0->sessions;
2314 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2316 /* verify speculative enqueue, maybe switch current next frame */
2317 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2318 to_next, n_left_to_next,
2322 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2325 vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2326 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2328 return frame->n_vectors;
2331 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2332 .function = snat_det_out2in_node_fn,
2333 .name = "nat44-det-out2in",
2334 .vector_size = sizeof (u32),
2335 .format_trace = format_snat_out2in_trace,
2336 .type = VLIB_NODE_TYPE_INTERNAL,
2338 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2339 .error_strings = snat_out2in_error_strings,
2341 .runtime_data_bytes = sizeof (snat_runtime_t),
2343 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2345 /* edit / add dispositions here */
2347 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2348 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2349 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2350 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2353 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2356 * Get address and port values to be used for ICMP packet translation
2357 * and create session if needed
2359 * @param[in,out] sm NAT main
2360 * @param[in,out] node NAT node runtime
2361 * @param[in] thread_index thread index
2362 * @param[in,out] b0 buffer containing packet to be translated
2363 * @param[out] p_proto protocol used for matching
2364 * @param[out] p_value address and port after NAT translation
2365 * @param[out] p_dont_translate if packet should not be translated
2366 * @param d optional parameter
2367 * @param e optional parameter
2369 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2370 u32 thread_index, vlib_buffer_t *b0,
2371 ip4_header_t *ip0, u8 *p_proto,
2372 snat_session_key_t *p_value,
2373 u8 *p_dont_translate, void *d, void *e)
2375 icmp46_header_t *icmp0;
2378 snat_det_out_key_t key0;
2379 u8 dont_translate = 0;
2381 icmp_echo_header_t *echo0, *inner_echo0 = 0;
2382 ip4_header_t *inner_ip0;
2383 void *l4_header = 0;
2384 icmp46_header_t *inner_icmp0;
2385 snat_det_map_t * dm0 = 0;
2386 ip4_address_t new_addr0 = {{0}};
2387 snat_det_session_t * ses0 = 0;
2388 ip4_address_t out_addr;
2390 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2391 echo0 = (icmp_echo_header_t *)(icmp0+1);
2392 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2394 if (!icmp_is_error_message (icmp0))
2396 protocol = SNAT_PROTOCOL_ICMP;
2397 key0.ext_host_addr = ip0->src_address;
2398 key0.ext_host_port = 0;
2399 key0.out_port = echo0->identifier;
2400 out_addr = ip0->dst_address;
2404 inner_ip0 = (ip4_header_t *)(echo0+1);
2405 l4_header = ip4_next_header (inner_ip0);
2406 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2407 key0.ext_host_addr = inner_ip0->dst_address;
2408 out_addr = inner_ip0->src_address;
2411 case SNAT_PROTOCOL_ICMP:
2412 inner_icmp0 = (icmp46_header_t*)l4_header;
2413 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2414 key0.ext_host_port = 0;
2415 key0.out_port = inner_echo0->identifier;
2417 case SNAT_PROTOCOL_UDP:
2418 case SNAT_PROTOCOL_TCP:
2419 key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2420 key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2423 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2424 next0 = SNAT_OUT2IN_NEXT_DROP;
2429 dm0 = snat_det_map_by_out(sm, &out_addr);
2430 if (PREDICT_FALSE(!dm0))
2432 /* Don't NAT packet aimed at the intfc address */
2433 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2434 ip0->dst_address.as_u32)))
2439 clib_warning("unknown dst address: %U",
2440 format_ip4_address, &ip0->dst_address);
2444 snat_det_reverse(dm0, &ip0->dst_address,
2445 clib_net_to_host_u16(key0.out_port), &new_addr0);
2447 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2448 if (PREDICT_FALSE(!ses0))
2450 /* Don't NAT packet aimed at the intfc address */
2451 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2452 ip0->dst_address.as_u32)))
2457 clib_warning("no match src %U:%d dst %U:%d for user %U",
2458 format_ip4_address, &key0.ext_host_addr,
2459 clib_net_to_host_u16 (key0.ext_host_port),
2460 format_ip4_address, &out_addr,
2461 clib_net_to_host_u16 (key0.out_port),
2462 format_ip4_address, &new_addr0);
2463 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2464 next0 = SNAT_OUT2IN_NEXT_DROP;
2468 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2469 !icmp_is_error_message (icmp0)))
2471 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2472 next0 = SNAT_OUT2IN_NEXT_DROP;
2479 *p_proto = protocol;
2482 p_value->addr = new_addr0;
2483 p_value->fib_index = sm->inside_fib_index;
2484 p_value->port = ses0->in_port;
2486 *p_dont_translate = dont_translate;
2488 *(snat_det_session_t**)d = ses0;
2490 *(snat_det_map_t**)e = dm0;
2494 /**********************/
2495 /*** worker handoff ***/
2496 /**********************/
2498 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2499 vlib_node_runtime_t * node,
2500 vlib_frame_t * frame)
2502 snat_main_t *sm = &snat_main;
2503 vlib_thread_main_t *tm = vlib_get_thread_main ();
2504 u32 n_left_from, *from, *to_next = 0;
2505 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2506 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2508 vlib_frame_queue_elt_t *hf = 0;
2509 vlib_frame_t *f = 0;
2511 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2512 u32 next_worker_index = 0;
2513 u32 current_worker_index = ~0;
2514 u32 thread_index = vlib_get_thread_index ();
2516 ASSERT (vec_len (sm->workers));
2518 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2520 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2522 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2523 sm->first_worker_index + sm->num_workers - 1,
2524 (vlib_frame_queue_t *) (~0));
2527 from = vlib_frame_vector_args (frame);
2528 n_left_from = frame->n_vectors;
2530 while (n_left_from > 0)
2543 b0 = vlib_get_buffer (vm, bi0);
2545 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2546 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2548 ip0 = vlib_buffer_get_current (b0);
2550 next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2552 if (PREDICT_FALSE (next_worker_index != thread_index))
2556 if (next_worker_index != current_worker_index)
2559 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2561 hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2563 handoff_queue_elt_by_worker_index);
2565 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2566 to_next_worker = &hf->buffer_index[hf->n_vectors];
2567 current_worker_index = next_worker_index;
2570 /* enqueue to correct worker thread */
2571 to_next_worker[0] = bi0;
2573 n_left_to_next_worker--;
2575 if (n_left_to_next_worker == 0)
2577 hf->n_vectors = VLIB_FRAME_SIZE;
2578 vlib_put_frame_queue_elt (hf);
2579 current_worker_index = ~0;
2580 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2587 /* if this is 1st frame */
2590 f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2591 to_next = vlib_frame_vector_args (f);
2599 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2600 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2602 snat_out2in_worker_handoff_trace_t *t =
2603 vlib_add_trace (vm, node, b0, sizeof (*t));
2604 t->next_worker_index = next_worker_index;
2605 t->do_handoff = do_handoff;
2610 vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2613 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2615 /* Ship frames to the worker nodes */
2616 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2618 if (handoff_queue_elt_by_worker_index[i])
2620 hf = handoff_queue_elt_by_worker_index[i];
2622 * It works better to let the handoff node
2623 * rate-adapt, always ship the handoff queue element.
2625 if (1 || hf->n_vectors == hf->last_n_vectors)
2627 vlib_put_frame_queue_elt (hf);
2628 handoff_queue_elt_by_worker_index[i] = 0;
2631 hf->last_n_vectors = hf->n_vectors;
2633 congested_handoff_queue_by_worker_index[i] =
2634 (vlib_frame_queue_t *) (~0);
2637 current_worker_index = ~0;
2638 return frame->n_vectors;
2641 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2642 .function = snat_out2in_worker_handoff_fn,
2643 .name = "nat44-out2in-worker-handoff",
2644 .vector_size = sizeof (u32),
2645 .format_trace = format_snat_out2in_worker_handoff_trace,
2646 .type = VLIB_NODE_TYPE_INTERNAL,
2655 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2658 snat_out2in_fast_node_fn (vlib_main_t * vm,
2659 vlib_node_runtime_t * node,
2660 vlib_frame_t * frame)
2662 u32 n_left_from, * from, * to_next;
2663 snat_out2in_next_t next_index;
2664 u32 pkts_processed = 0;
2665 snat_main_t * sm = &snat_main;
2667 from = vlib_frame_vector_args (frame);
2668 n_left_from = frame->n_vectors;
2669 next_index = node->cached_next_index;
2671 while (n_left_from > 0)
2675 vlib_get_next_frame (vm, node, next_index,
2676 to_next, n_left_to_next);
2678 while (n_left_from > 0 && n_left_to_next > 0)
2682 u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2686 u32 new_addr0, old_addr0;
2687 u16 new_port0, old_port0;
2688 udp_header_t * udp0;
2689 tcp_header_t * tcp0;
2690 icmp46_header_t * icmp0;
2691 snat_session_key_t key0, sm0;
2695 /* speculatively enqueue b0 to the current next frame */
2701 n_left_to_next -= 1;
2703 b0 = vlib_get_buffer (vm, bi0);
2705 ip0 = vlib_buffer_get_current (b0);
2706 udp0 = ip4_next_header (ip0);
2707 tcp0 = (tcp_header_t *) udp0;
2708 icmp0 = (icmp46_header_t *) udp0;
2710 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2711 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2713 vnet_feature_next (sw_if_index0, &next0, b0);
2715 if (PREDICT_FALSE(ip0->ttl == 1))
2717 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2718 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2719 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2721 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2725 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2727 if (PREDICT_FALSE (proto0 == ~0))
2730 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2732 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2733 rx_fib_index0, node, next0, ~0, 0, 0);
2737 key0.addr = ip0->dst_address;
2738 key0.port = udp0->dst_port;
2739 key0.fib_index = rx_fib_index0;
2741 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2743 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2747 new_addr0 = sm0.addr.as_u32;
2748 new_port0 = sm0.port;
2749 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2750 old_addr0 = ip0->dst_address.as_u32;
2751 ip0->dst_address.as_u32 = new_addr0;
2753 sum0 = ip0->checksum;
2754 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2756 dst_address /* changed member */);
2757 ip0->checksum = ip_csum_fold (sum0);
2759 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2761 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2763 old_port0 = tcp0->dst_port;
2764 tcp0->dst_port = new_port0;
2766 sum0 = tcp0->checksum;
2767 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2769 dst_address /* changed member */);
2771 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2772 ip4_header_t /* cheat */,
2773 length /* changed member */);
2774 tcp0->checksum = ip_csum_fold(sum0);
2778 old_port0 = udp0->dst_port;
2779 udp0->dst_port = new_port0;
2785 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2787 sum0 = tcp0->checksum;
2788 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2790 dst_address /* changed member */);
2792 tcp0->checksum = ip_csum_fold(sum0);
2798 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2799 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2801 snat_out2in_trace_t *t =
2802 vlib_add_trace (vm, node, b0, sizeof (*t));
2803 t->sw_if_index = sw_if_index0;
2804 t->next_index = next0;
2807 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2809 /* verify speculative enqueue, maybe switch current next frame */
2810 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2811 to_next, n_left_to_next,
2815 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2818 vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2819 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2821 return frame->n_vectors;
2824 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2825 .function = snat_out2in_fast_node_fn,
2826 .name = "nat44-out2in-fast",
2827 .vector_size = sizeof (u32),
2828 .format_trace = format_snat_out2in_fast_trace,
2829 .type = VLIB_NODE_TYPE_INTERNAL,
2831 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2832 .error_strings = snat_out2in_error_strings,
2834 .runtime_data_bytes = sizeof (snat_runtime_t),
2836 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2838 /* edit / add dispositions here */
2840 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2841 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2842 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2843 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
2846 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);