2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
26 #include <nat/nat_ipfix_logging.h>
27 #include <nat/nat_det.h>
29 #include <vppinfra/hash.h>
30 #include <vppinfra/error.h>
31 #include <vppinfra/elog.h>
37 } snat_out2in_trace_t;
40 u32 next_worker_index;
42 } snat_out2in_worker_handoff_trace_t;
44 /* packet trace format function */
45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
47 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
48 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
49 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
51 s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
52 t->sw_if_index, t->next_index, t->session_index);
56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
58 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
59 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
60 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
62 s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
63 t->sw_if_index, t->next_index);
67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
69 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
70 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
71 snat_out2in_worker_handoff_trace_t * t =
72 va_arg (*args, snat_out2in_worker_handoff_trace_t *);
75 m = t->do_handoff ? "next worker" : "same worker";
76 s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
81 vlib_node_registration_t snat_out2in_node;
82 vlib_node_registration_t snat_out2in_fast_node;
83 vlib_node_registration_t snat_out2in_worker_handoff_node;
84 vlib_node_registration_t snat_det_out2in_node;
86 #define foreach_snat_out2in_error \
87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
88 _(OUT2IN_PACKETS, "Good out2in packets processed") \
89 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
90 _(NO_TRANSLATION, "No translation") \
91 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")
94 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
95 foreach_snat_out2in_error
98 } snat_out2in_error_t;
100 static char * snat_out2in_error_strings[] = {
101 #define _(sym,string) string,
102 foreach_snat_out2in_error
107 SNAT_OUT2IN_NEXT_DROP,
108 SNAT_OUT2IN_NEXT_LOOKUP,
109 SNAT_OUT2IN_NEXT_ICMP_ERROR,
111 } snat_out2in_next_t;
114 * @brief Create session for static mapping.
116 * Create NAT session initiated by host from external network with static
119 * @param sm NAT main.
120 * @param b0 Vlib buffer.
121 * @param in2out In2out NAT44 session key.
122 * @param out2in Out2in NAT44 session key.
123 * @param node Vlib node.
125 * @returns SNAT session if successfully created otherwise 0.
127 static inline snat_session_t *
128 create_session_for_static_mapping (snat_main_t *sm,
130 snat_session_key_t in2out,
131 snat_session_key_t out2in,
132 vlib_node_runtime_t * node,
136 snat_user_key_t user_key;
138 clib_bihash_kv_8_8_t kv0, value0;
139 dlist_elt_t * per_user_translation_list_elt;
140 dlist_elt_t * per_user_list_head_elt;
143 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
145 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
149 ip0 = vlib_buffer_get_current (b0);
151 user_key.addr = in2out.addr;
152 user_key.fib_index = in2out.fib_index;
153 kv0.key = user_key.as_u64;
155 /* Ever heard of the "user" = inside ip4 address before? */
156 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].user_hash,
159 /* no, make a new one */
160 pool_get (sm->per_thread_data[thread_index].users, u);
161 memset (u, 0, sizeof (*u));
162 u->addr = in2out.addr;
163 u->fib_index = in2out.fib_index;
165 pool_get (sm->per_thread_data[thread_index].list_pool,
166 per_user_list_head_elt);
168 u->sessions_per_user_list_head_index = per_user_list_head_elt -
169 sm->per_thread_data[thread_index].list_pool;
171 clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
172 u->sessions_per_user_list_head_index);
174 kv0.value = u - sm->per_thread_data[thread_index].users;
177 clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].user_hash,
178 &kv0, 1 /* is_add */);
180 /* add non-traslated packets worker lookup */
181 kv0.value = thread_index;
182 clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
186 u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
190 pool_get (sm->per_thread_data[thread_index].sessions, s);
191 memset (s, 0, sizeof (*s));
193 s->outside_address_index = ~0;
194 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
195 s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
196 u->nstaticsessions++;
198 /* Create list elts */
199 pool_get (sm->per_thread_data[thread_index].list_pool,
200 per_user_translation_list_elt);
201 clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
202 per_user_translation_list_elt -
203 sm->per_thread_data[thread_index].list_pool);
205 per_user_translation_list_elt->value =
206 s - sm->per_thread_data[thread_index].sessions;
208 per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
209 s->per_user_list_head_index = u->sessions_per_user_list_head_index;
211 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
212 s->per_user_list_head_index,
213 per_user_translation_list_elt -
214 sm->per_thread_data[thread_index].list_pool);
218 s->in2out.protocol = out2in.protocol;
220 /* Add to translation hashes */
221 kv0.key = s->in2out.as_u64;
222 kv0.value = s - sm->per_thread_data[thread_index].sessions;
223 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
225 clib_warning ("in2out key add failed");
227 kv0.key = s->out2in.as_u64;
228 kv0.value = s - sm->per_thread_data[thread_index].sessions;
230 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
232 clib_warning ("out2in key add failed");
235 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
236 s->out2in.addr.as_u32,
240 s->in2out.fib_index);
245 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
246 snat_session_key_t *p_key0)
248 icmp46_header_t *icmp0;
249 snat_session_key_t key0;
250 icmp_echo_header_t *echo0, *inner_echo0 = 0;
251 ip4_header_t *inner_ip0;
253 icmp46_header_t *inner_icmp0;
255 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
256 echo0 = (icmp_echo_header_t *)(icmp0+1);
258 if (!icmp_is_error_message (icmp0))
260 key0.protocol = SNAT_PROTOCOL_ICMP;
261 key0.addr = ip0->dst_address;
262 key0.port = echo0->identifier;
266 inner_ip0 = (ip4_header_t *)(echo0+1);
267 l4_header = ip4_next_header (inner_ip0);
268 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
269 key0.addr = inner_ip0->src_address;
270 switch (key0.protocol)
272 case SNAT_PROTOCOL_ICMP:
273 inner_icmp0 = (icmp46_header_t*)l4_header;
274 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
275 key0.port = inner_echo0->identifier;
277 case SNAT_PROTOCOL_UDP:
278 case SNAT_PROTOCOL_TCP:
279 key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
282 return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
286 return -1; /* success */
290 * Get address and port values to be used for ICMP packet translation
291 * and create session if needed
293 * @param[in,out] sm NAT main
294 * @param[in,out] node NAT node runtime
295 * @param[in] thread_index thread index
296 * @param[in,out] b0 buffer containing packet to be translated
297 * @param[out] p_proto protocol used for matching
298 * @param[out] p_value address and port after NAT translation
299 * @param[out] p_dont_translate if packet should not be translated
300 * @param d optional parameter
301 * @param e optional parameter
303 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
304 u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
305 snat_session_key_t *p_value,
306 u8 *p_dont_translate, void *d, void *e)
309 icmp46_header_t *icmp0;
312 snat_session_key_t key0;
313 snat_session_key_t sm0;
314 snat_session_t *s0 = 0;
315 u8 dont_translate = 0;
316 clib_bihash_kv_8_8_t kv0, value0;
321 ip0 = vlib_buffer_get_current (b0);
322 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
323 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
324 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
328 err = icmp_get_key (ip0, &key0);
331 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
332 next0 = SNAT_OUT2IN_NEXT_DROP;
335 key0.fib_index = rx_fib_index0;
337 kv0.key = key0.as_u64;
339 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
342 /* Try to match static mapping by external address and port,
343 destination address and port in packet */
344 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
346 /* Don't NAT packet aimed at the intfc address */
347 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
348 ip0->dst_address.as_u32)))
353 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
354 next0 = SNAT_OUT2IN_NEXT_DROP;
358 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
359 (icmp0->type != ICMP4_echo_request || !is_addr_only)))
361 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
362 next0 = SNAT_OUT2IN_NEXT_DROP;
366 /* Create session initiated by host from external network */
367 s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
372 next0 = SNAT_OUT2IN_NEXT_DROP;
378 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
379 icmp0->type != ICMP4_echo_request &&
380 !icmp_is_error_message (icmp0)))
382 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
383 next0 = SNAT_OUT2IN_NEXT_DROP;
387 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
392 *p_proto = key0.protocol;
394 *p_value = s0->in2out;
395 *p_dont_translate = dont_translate;
397 *(snat_session_t**)d = s0;
402 * Get address and port values to be used for ICMP packet translation
404 * @param[in] sm NAT main
405 * @param[in,out] node NAT node runtime
406 * @param[in] thread_index thread index
407 * @param[in,out] b0 buffer containing packet to be translated
408 * @param[out] p_proto protocol used for matching
409 * @param[out] p_value address and port after NAT translation
410 * @param[out] p_dont_translate if packet should not be translated
411 * @param d optional parameter
412 * @param e optional parameter
414 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
415 u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
416 snat_session_key_t *p_value,
417 u8 *p_dont_translate, void *d, void *e)
420 icmp46_header_t *icmp0;
423 snat_session_key_t key0;
424 snat_session_key_t sm0;
425 u8 dont_translate = 0;
430 ip0 = vlib_buffer_get_current (b0);
431 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
432 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
433 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
435 err = icmp_get_key (ip0, &key0);
438 b0->error = node->errors[err];
439 next0 = SNAT_OUT2IN_NEXT_DROP;
442 key0.fib_index = rx_fib_index0;
444 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
446 /* Don't NAT packet aimed at the intfc address */
447 if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
452 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
453 next0 = SNAT_OUT2IN_NEXT_DROP;
457 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
458 (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
459 !icmp_is_error_message (icmp0)))
461 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
462 next0 = SNAT_OUT2IN_NEXT_DROP;
469 *p_proto = key0.protocol;
470 *p_dont_translate = dont_translate;
474 static inline u32 icmp_out2in (snat_main_t *sm,
477 icmp46_header_t * icmp0,
480 vlib_node_runtime_t * node,
486 snat_session_key_t sm0;
488 icmp_echo_header_t *echo0, *inner_echo0 = 0;
489 ip4_header_t *inner_ip0 = 0;
491 icmp46_header_t *inner_icmp0;
493 u32 new_addr0, old_addr0;
494 u16 old_id0, new_id0;
499 echo0 = (icmp_echo_header_t *)(icmp0+1);
501 next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
502 &protocol, &sm0, &dont_translate, d, e);
505 if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
508 sum0 = ip_incremental_checksum (0, icmp0,
509 ntohs(ip0->length) - ip4_header_bytes (ip0));
510 checksum0 = ~ip_csum_fold (sum0);
511 if (checksum0 != 0 && checksum0 != 0xffff)
513 next0 = SNAT_OUT2IN_NEXT_DROP;
517 old_addr0 = ip0->dst_address.as_u32;
518 new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
519 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
521 sum0 = ip0->checksum;
522 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
523 dst_address /* changed member */);
524 ip0->checksum = ip_csum_fold (sum0);
526 if (!icmp_is_error_message (icmp0))
529 if (PREDICT_FALSE(new_id0 != echo0->identifier))
531 old_id0 = echo0->identifier;
533 echo0->identifier = new_id0;
535 sum0 = icmp0->checksum;
536 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
537 identifier /* changed member */);
538 icmp0->checksum = ip_csum_fold (sum0);
543 inner_ip0 = (ip4_header_t *)(echo0+1);
544 l4_header = ip4_next_header (inner_ip0);
546 if (!ip4_header_checksum_is_valid (inner_ip0))
548 next0 = SNAT_OUT2IN_NEXT_DROP;
552 old_addr0 = inner_ip0->src_address.as_u32;
553 inner_ip0->src_address = sm0.addr;
554 new_addr0 = inner_ip0->src_address.as_u32;
556 sum0 = icmp0->checksum;
557 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
558 src_address /* changed member */);
559 icmp0->checksum = ip_csum_fold (sum0);
563 case SNAT_PROTOCOL_ICMP:
564 inner_icmp0 = (icmp46_header_t*)l4_header;
565 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
567 old_id0 = inner_echo0->identifier;
569 inner_echo0->identifier = new_id0;
571 sum0 = icmp0->checksum;
572 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
574 icmp0->checksum = ip_csum_fold (sum0);
576 case SNAT_PROTOCOL_UDP:
577 case SNAT_PROTOCOL_TCP:
578 old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
580 ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
582 sum0 = icmp0->checksum;
583 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
585 icmp0->checksum = ip_csum_fold (sum0);
597 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
600 icmp46_header_t * icmp0,
603 vlib_node_runtime_t * node,
606 snat_session_t ** p_s0)
608 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
609 next0, thread_index, p_s0, 0);
610 snat_session_t * s0 = *p_s0;
611 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
614 s0->last_heard = now;
616 s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
617 /* Per-user LRU list maintenance for dynamic translation */
618 if (!snat_is_session_static (s0))
620 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
622 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
623 s0->per_user_list_head_index,
630 static snat_session_t *
631 snat_out2in_unknown_proto (snat_main_t *sm,
638 vlib_node_runtime_t * node)
640 clib_bihash_kv_8_8_t kv, value;
641 clib_bihash_kv_16_8_t s_kv, s_value;
642 snat_static_mapping_t *m;
643 snat_session_key_t m_key;
644 u32 old_addr, new_addr;
646 nat_ed_ses_key_t key;
648 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
649 snat_user_key_t u_key;
651 dlist_elt_t *head, *elt;
653 old_addr = ip->dst_address.as_u32;
655 key.l_addr = ip->dst_address;
656 key.r_addr = ip->src_address;
657 key.fib_index = rx_fib_index;
658 key.proto = ip->protocol;
661 s_kv.key[0] = key.as_u64[0];
662 s_kv.key[1] = key.as_u64[1];
664 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
666 s = pool_elt_at_index (tsm->sessions, s_value.value);
667 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
671 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
673 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
677 m_key.addr = ip->dst_address;
680 m_key.fib_index = rx_fib_index;
681 kv.key = m_key.as_u64;
682 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
684 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
688 m = pool_elt_at_index (sm->static_mappings, value.value);
690 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
692 u_key.addr = ip->src_address;
693 u_key.fib_index = m->fib_index;
694 kv.key = u_key.as_u64;
696 /* Ever heard of the "user" = src ip4 address before? */
697 if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
699 /* no, make a new one */
700 pool_get (tsm->users, u);
701 memset (u, 0, sizeof (*u));
702 u->addr = ip->src_address;
703 u->fib_index = rx_fib_index;
705 pool_get (tsm->list_pool, head);
706 u->sessions_per_user_list_head_index = head - tsm->list_pool;
708 clib_dlist_init (tsm->list_pool,
709 u->sessions_per_user_list_head_index);
711 kv.value = u - tsm->users;
714 clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1);
718 u = pool_elt_at_index (tsm->users, value.value);
721 /* Create a new session */
722 pool_get (tsm->sessions, s);
723 memset (s, 0, sizeof (*s));
725 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
726 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
727 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
728 s->outside_address_index = ~0;
729 s->out2in.addr.as_u32 = old_addr;
730 s->out2in.fib_index = rx_fib_index;
731 s->in2out.addr.as_u32 = new_addr;
732 s->in2out.fib_index = m->fib_index;
733 s->in2out.port = s->out2in.port = ip->protocol;
734 u->nstaticsessions++;
736 /* Create list elts */
737 pool_get (tsm->list_pool, elt);
738 clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
739 elt->value = s - tsm->sessions;
740 s->per_user_index = elt - tsm->list_pool;
741 s->per_user_list_head_index = u->sessions_per_user_list_head_index;
742 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
745 /* Add to lookup tables */
746 s_kv.value = s - tsm->sessions;
747 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
748 clib_warning ("out2in key add failed");
750 key.l_addr = ip->dst_address;
751 key.fib_index = m->fib_index;
752 s_kv.key[0] = key.as_u64[0];
753 s_kv.key[1] = key.as_u64[1];
754 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
755 clib_warning ("in2out key add failed");
758 /* Update IP checksum */
760 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
761 ip->checksum = ip_csum_fold (sum);
763 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
768 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
769 /* Per-user LRU list maintenance */
770 clib_dlist_remove (tsm->list_pool, s->per_user_index);
771 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
777 static snat_session_t *
778 snat_out2in_lb (snat_main_t *sm,
785 vlib_node_runtime_t * node)
787 nat_ed_ses_key_t key;
788 clib_bihash_kv_16_8_t s_kv, s_value;
789 udp_header_t *udp = ip4_next_header (ip);
790 tcp_header_t *tcp = (tcp_header_t *) udp;
791 snat_session_t *s = 0;
792 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
793 snat_session_key_t e_key, l_key;
794 clib_bihash_kv_8_8_t kv, value;
795 u32 old_addr, new_addr;
796 u32 proto = ip_proto_to_snat_proto (ip->protocol);
797 u16 new_port, old_port;
799 snat_user_key_t u_key;
801 dlist_elt_t *head, *elt;
803 old_addr = ip->dst_address.as_u32;
805 key.l_addr = ip->dst_address;
806 key.r_addr = ip->src_address;
807 key.fib_index = rx_fib_index;
808 key.proto = ip->protocol;
810 key.l_port = udp->dst_port;
811 s_kv.key[0] = key.as_u64[0];
812 s_kv.key[1] = key.as_u64[1];
814 if (!clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
816 s = pool_elt_at_index (tsm->sessions, s_value.value);
820 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
822 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
826 e_key.addr = ip->dst_address;
827 e_key.port = udp->dst_port;
828 e_key.protocol = proto;
829 e_key.fib_index = rx_fib_index;
830 if (snat_static_mapping_match(sm, e_key, &l_key, 1, 0))
833 u_key.addr = l_key.addr;
834 u_key.fib_index = l_key.fib_index;
835 kv.key = u_key.as_u64;
837 /* Ever heard of the "user" = src ip4 address before? */
838 if (clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
840 /* no, make a new one */
841 pool_get (tsm->users, u);
842 memset (u, 0, sizeof (*u));
843 u->addr = l_key.addr;
844 u->fib_index = l_key.fib_index;
846 pool_get (tsm->list_pool, head);
847 u->sessions_per_user_list_head_index = head - tsm->list_pool;
849 clib_dlist_init (tsm->list_pool,
850 u->sessions_per_user_list_head_index);
852 kv.value = u - tsm->users;
855 if (clib_bihash_add_del_8_8 (&tsm->user_hash, &kv, 1))
856 clib_warning ("user key add failed");
860 u = pool_elt_at_index (tsm->users, value.value);
863 /* Create a new session */
864 pool_get (tsm->sessions, s);
865 memset (s, 0, sizeof (*s));
867 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
868 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
869 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
870 s->outside_address_index = ~0;
873 u->nstaticsessions++;
875 /* Create list elts */
876 pool_get (tsm->list_pool, elt);
877 clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
878 elt->value = s - tsm->sessions;
879 s->per_user_index = elt - tsm->list_pool;
880 s->per_user_list_head_index = u->sessions_per_user_list_head_index;
881 clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
884 /* Add to lookup tables */
885 s_kv.value = s - tsm->sessions;
886 if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
887 clib_warning ("out2in-ed key add failed");
889 key.l_addr = l_key.addr;
890 key.fib_index = l_key.fib_index;
891 key.l_port = l_key.port;
892 s_kv.key[0] = key.as_u64[0];
893 s_kv.key[1] = key.as_u64[1];
894 if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
895 clib_warning ("in2out-ed key add failed");
898 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
900 /* Update IP checksum */
902 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
903 ip->checksum = ip_csum_fold (sum);
905 if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
907 old_port = tcp->dst_port;
908 tcp->dst_port = s->in2out.port;
909 new_port = tcp->dst_port;
912 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
913 sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
914 tcp->checksum = ip_csum_fold(sum);
918 udp->dst_port = s->in2out.port;
922 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
927 s->total_bytes += vlib_buffer_length_in_chain (vm, b);
932 snat_out2in_node_fn (vlib_main_t * vm,
933 vlib_node_runtime_t * node,
934 vlib_frame_t * frame)
936 u32 n_left_from, * from, * to_next;
937 snat_out2in_next_t next_index;
938 u32 pkts_processed = 0;
939 snat_main_t * sm = &snat_main;
940 f64 now = vlib_time_now (vm);
941 u32 thread_index = vlib_get_thread_index ();
943 from = vlib_frame_vector_args (frame);
944 n_left_from = frame->n_vectors;
945 next_index = node->cached_next_index;
947 while (n_left_from > 0)
951 vlib_get_next_frame (vm, node, next_index,
952 to_next, n_left_to_next);
954 while (n_left_from >= 4 && n_left_to_next >= 2)
957 vlib_buffer_t * b0, * b1;
958 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
959 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
960 u32 sw_if_index0, sw_if_index1;
961 ip4_header_t * ip0, *ip1;
962 ip_csum_t sum0, sum1;
963 u32 new_addr0, old_addr0;
964 u16 new_port0, old_port0;
965 u32 new_addr1, old_addr1;
966 u16 new_port1, old_port1;
967 udp_header_t * udp0, * udp1;
968 tcp_header_t * tcp0, * tcp1;
969 icmp46_header_t * icmp0, * icmp1;
970 snat_session_key_t key0, key1, sm0, sm1;
971 u32 rx_fib_index0, rx_fib_index1;
973 snat_session_t * s0 = 0, * s1 = 0;
974 clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
976 /* Prefetch next iteration. */
978 vlib_buffer_t * p2, * p3;
980 p2 = vlib_get_buffer (vm, from[2]);
981 p3 = vlib_get_buffer (vm, from[3]);
983 vlib_prefetch_buffer_header (p2, LOAD);
984 vlib_prefetch_buffer_header (p3, LOAD);
986 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
987 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
990 /* speculatively enqueue b0 and b1 to the current next frame */
991 to_next[0] = bi0 = from[0];
992 to_next[1] = bi1 = from[1];
998 b0 = vlib_get_buffer (vm, bi0);
999 b1 = vlib_get_buffer (vm, bi1);
1001 vnet_buffer (b0)->snat.flags = 0;
1002 vnet_buffer (b1)->snat.flags = 0;
1004 ip0 = vlib_buffer_get_current (b0);
1005 udp0 = ip4_next_header (ip0);
1006 tcp0 = (tcp_header_t *) udp0;
1007 icmp0 = (icmp46_header_t *) udp0;
1009 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1010 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1013 if (PREDICT_FALSE(ip0->ttl == 1))
1015 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1016 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1017 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1019 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1023 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1025 if (PREDICT_FALSE (proto0 == ~0))
1027 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1028 thread_index, now, vm, node);
1030 next0 = SNAT_OUT2IN_NEXT_DROP;
1034 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1036 next0 = icmp_out2in_slow_path
1037 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1038 next0, now, thread_index, &s0);
1042 key0.addr = ip0->dst_address;
1043 key0.port = udp0->dst_port;
1044 key0.protocol = proto0;
1045 key0.fib_index = rx_fib_index0;
1047 kv0.key = key0.as_u64;
1049 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1052 /* Try to match static mapping by external address and port,
1053 destination address and port in packet */
1054 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1056 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1058 * Send DHCP packets to the ipv4 stack, or we won't
1059 * be able to use dhcp client on the outside interface
1061 if (proto0 != SNAT_PROTOCOL_UDP
1063 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1064 next0 = SNAT_OUT2IN_NEXT_DROP;
1068 /* Create session initiated by host from external network */
1069 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1073 next0 = SNAT_OUT2IN_NEXT_DROP;
1079 if (PREDICT_FALSE (value0.value == ~0ULL))
1081 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1084 next0 = SNAT_OUT2IN_NEXT_DROP;
1089 s0 = pool_elt_at_index (
1090 sm->per_thread_data[thread_index].sessions,
1095 old_addr0 = ip0->dst_address.as_u32;
1096 ip0->dst_address = s0->in2out.addr;
1097 new_addr0 = ip0->dst_address.as_u32;
1098 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1100 sum0 = ip0->checksum;
1101 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1103 dst_address /* changed member */);
1104 ip0->checksum = ip_csum_fold (sum0);
1106 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1108 old_port0 = tcp0->dst_port;
1109 tcp0->dst_port = s0->in2out.port;
1110 new_port0 = tcp0->dst_port;
1112 sum0 = tcp0->checksum;
1113 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1115 dst_address /* changed member */);
1117 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1118 ip4_header_t /* cheat */,
1119 length /* changed member */);
1120 tcp0->checksum = ip_csum_fold(sum0);
1124 old_port0 = udp0->dst_port;
1125 udp0->dst_port = s0->in2out.port;
1130 s0->last_heard = now;
1132 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1133 /* Per-user LRU list maintenance for dynamic translation */
1134 if (!snat_is_session_static (s0))
1136 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1137 s0->per_user_index);
1138 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1139 s0->per_user_list_head_index,
1140 s0->per_user_index);
1144 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1145 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1147 snat_out2in_trace_t *t =
1148 vlib_add_trace (vm, node, b0, sizeof (*t));
1149 t->sw_if_index = sw_if_index0;
1150 t->next_index = next0;
1151 t->session_index = ~0;
1153 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1156 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1159 ip1 = vlib_buffer_get_current (b1);
1160 udp1 = ip4_next_header (ip1);
1161 tcp1 = (tcp_header_t *) udp1;
1162 icmp1 = (icmp46_header_t *) udp1;
1164 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1165 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1168 if (PREDICT_FALSE(ip1->ttl == 1))
1170 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1171 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1172 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1174 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1178 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1180 if (PREDICT_FALSE (proto1 == ~0))
1182 s1 = snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
1183 thread_index, now, vm, node);
1185 next1 = SNAT_OUT2IN_NEXT_DROP;
1189 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1191 next1 = icmp_out2in_slow_path
1192 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1193 next1, now, thread_index, &s1);
1197 key1.addr = ip1->dst_address;
1198 key1.port = udp1->dst_port;
1199 key1.protocol = proto1;
1200 key1.fib_index = rx_fib_index1;
1202 kv1.key = key1.as_u64;
1204 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1207 /* Try to match static mapping by external address and port,
1208 destination address and port in packet */
1209 if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
1211 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1213 * Send DHCP packets to the ipv4 stack, or we won't
1214 * be able to use dhcp client on the outside interface
1216 if (proto1 != SNAT_PROTOCOL_UDP
1218 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1219 next1 = SNAT_OUT2IN_NEXT_DROP;
1223 /* Create session initiated by host from external network */
1224 s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1228 next1 = SNAT_OUT2IN_NEXT_DROP;
1234 if (PREDICT_FALSE (value1.value == ~0ULL))
1236 s1 = snat_out2in_lb(sm, b1, ip1, rx_fib_index1, thread_index,
1239 next1 = SNAT_OUT2IN_NEXT_DROP;
1244 s1 = pool_elt_at_index (
1245 sm->per_thread_data[thread_index].sessions,
1250 old_addr1 = ip1->dst_address.as_u32;
1251 ip1->dst_address = s1->in2out.addr;
1252 new_addr1 = ip1->dst_address.as_u32;
1253 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1255 sum1 = ip1->checksum;
1256 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1258 dst_address /* changed member */);
1259 ip1->checksum = ip_csum_fold (sum1);
1261 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1263 old_port1 = tcp1->dst_port;
1264 tcp1->dst_port = s1->in2out.port;
1265 new_port1 = tcp1->dst_port;
1267 sum1 = tcp1->checksum;
1268 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1270 dst_address /* changed member */);
1272 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1273 ip4_header_t /* cheat */,
1274 length /* changed member */);
1275 tcp1->checksum = ip_csum_fold(sum1);
1279 old_port1 = udp1->dst_port;
1280 udp1->dst_port = s1->in2out.port;
1285 s1->last_heard = now;
1287 s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1288 /* Per-user LRU list maintenance for dynamic translation */
1289 if (!snat_is_session_static (s1))
1291 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1292 s1->per_user_index);
1293 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1294 s1->per_user_list_head_index,
1295 s1->per_user_index);
1299 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1300 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1302 snat_out2in_trace_t *t =
1303 vlib_add_trace (vm, node, b1, sizeof (*t));
1304 t->sw_if_index = sw_if_index1;
1305 t->next_index = next1;
1306 t->session_index = ~0;
1308 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1311 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1313 /* verify speculative enqueues, maybe switch current next frame */
1314 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1315 to_next, n_left_to_next,
1316 bi0, bi1, next0, next1);
1319 while (n_left_from > 0 && n_left_to_next > 0)
1323 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1327 u32 new_addr0, old_addr0;
1328 u16 new_port0, old_port0;
1329 udp_header_t * udp0;
1330 tcp_header_t * tcp0;
1331 icmp46_header_t * icmp0;
1332 snat_session_key_t key0, sm0;
1335 snat_session_t * s0 = 0;
1336 clib_bihash_kv_8_8_t kv0, value0;
1338 /* speculatively enqueue b0 to the current next frame */
1344 n_left_to_next -= 1;
1346 b0 = vlib_get_buffer (vm, bi0);
1348 vnet_buffer (b0)->snat.flags = 0;
1350 ip0 = vlib_buffer_get_current (b0);
1351 udp0 = ip4_next_header (ip0);
1352 tcp0 = (tcp_header_t *) udp0;
1353 icmp0 = (icmp46_header_t *) udp0;
1355 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1356 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1359 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1361 if (PREDICT_FALSE (proto0 == ~0))
1363 s0 = snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
1364 thread_index, now, vm, node);
1366 next0 = SNAT_OUT2IN_NEXT_DROP;
1370 if (PREDICT_FALSE(ip0->ttl == 1))
1372 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1373 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1374 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1376 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1380 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1382 next0 = icmp_out2in_slow_path
1383 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1384 next0, now, thread_index, &s0);
1388 key0.addr = ip0->dst_address;
1389 key0.port = udp0->dst_port;
1390 key0.protocol = proto0;
1391 key0.fib_index = rx_fib_index0;
1393 kv0.key = key0.as_u64;
1395 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1398 /* Try to match static mapping by external address and port,
1399 destination address and port in packet */
1400 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1402 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1404 * Send DHCP packets to the ipv4 stack, or we won't
1405 * be able to use dhcp client on the outside interface
1407 if (proto0 != SNAT_PROTOCOL_UDP
1409 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1411 next0 = SNAT_OUT2IN_NEXT_DROP;
1415 /* Create session initiated by host from external network */
1416 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1420 next0 = SNAT_OUT2IN_NEXT_DROP;
1426 if (PREDICT_FALSE (value0.value == ~0ULL))
1428 s0 = snat_out2in_lb(sm, b0, ip0, rx_fib_index0, thread_index,
1431 next0 = SNAT_OUT2IN_NEXT_DROP;
1436 s0 = pool_elt_at_index (
1437 sm->per_thread_data[thread_index].sessions,
1442 old_addr0 = ip0->dst_address.as_u32;
1443 ip0->dst_address = s0->in2out.addr;
1444 new_addr0 = ip0->dst_address.as_u32;
1445 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1447 sum0 = ip0->checksum;
1448 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1450 dst_address /* changed member */);
1451 ip0->checksum = ip_csum_fold (sum0);
1453 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1455 old_port0 = tcp0->dst_port;
1456 tcp0->dst_port = s0->in2out.port;
1457 new_port0 = tcp0->dst_port;
1459 sum0 = tcp0->checksum;
1460 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1462 dst_address /* changed member */);
1464 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1465 ip4_header_t /* cheat */,
1466 length /* changed member */);
1467 tcp0->checksum = ip_csum_fold(sum0);
1471 old_port0 = udp0->dst_port;
1472 udp0->dst_port = s0->in2out.port;
1477 s0->last_heard = now;
1479 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1480 /* Per-user LRU list maintenance for dynamic translation */
1481 if (!snat_is_session_static (s0))
1483 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1484 s0->per_user_index);
1485 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1486 s0->per_user_list_head_index,
1487 s0->per_user_index);
1491 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1492 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1494 snat_out2in_trace_t *t =
1495 vlib_add_trace (vm, node, b0, sizeof (*t));
1496 t->sw_if_index = sw_if_index0;
1497 t->next_index = next0;
1498 t->session_index = ~0;
1500 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1503 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1505 /* verify speculative enqueue, maybe switch current next frame */
1506 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1507 to_next, n_left_to_next,
1511 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1514 vlib_node_increment_counter (vm, snat_out2in_node.index,
1515 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1517 return frame->n_vectors;
1520 VLIB_REGISTER_NODE (snat_out2in_node) = {
1521 .function = snat_out2in_node_fn,
1522 .name = "nat44-out2in",
1523 .vector_size = sizeof (u32),
1524 .format_trace = format_snat_out2in_trace,
1525 .type = VLIB_NODE_TYPE_INTERNAL,
1527 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1528 .error_strings = snat_out2in_error_strings,
1530 .runtime_data_bytes = sizeof (snat_runtime_t),
1532 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1534 /* edit / add dispositions here */
1536 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1537 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1538 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1541 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1543 /**************************/
1544 /*** deterministic mode ***/
1545 /**************************/
1547 snat_det_out2in_node_fn (vlib_main_t * vm,
1548 vlib_node_runtime_t * node,
1549 vlib_frame_t * frame)
1551 u32 n_left_from, * from, * to_next;
1552 snat_out2in_next_t next_index;
1553 u32 pkts_processed = 0;
1554 snat_main_t * sm = &snat_main;
1555 u32 thread_index = vlib_get_thread_index ();
1557 from = vlib_frame_vector_args (frame);
1558 n_left_from = frame->n_vectors;
1559 next_index = node->cached_next_index;
1561 while (n_left_from > 0)
1565 vlib_get_next_frame (vm, node, next_index,
1566 to_next, n_left_to_next);
1568 while (n_left_from >= 4 && n_left_to_next >= 2)
1571 vlib_buffer_t * b0, * b1;
1572 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1573 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1574 u32 sw_if_index0, sw_if_index1;
1575 ip4_header_t * ip0, * ip1;
1576 ip_csum_t sum0, sum1;
1577 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1578 u16 new_port0, old_port0, old_port1, new_port1;
1579 udp_header_t * udp0, * udp1;
1580 tcp_header_t * tcp0, * tcp1;
1582 snat_det_out_key_t key0, key1;
1583 snat_det_map_t * dm0, * dm1;
1584 snat_det_session_t * ses0 = 0, * ses1 = 0;
1585 u32 rx_fib_index0, rx_fib_index1;
1586 icmp46_header_t * icmp0, * icmp1;
1588 /* Prefetch next iteration. */
1590 vlib_buffer_t * p2, * p3;
1592 p2 = vlib_get_buffer (vm, from[2]);
1593 p3 = vlib_get_buffer (vm, from[3]);
1595 vlib_prefetch_buffer_header (p2, LOAD);
1596 vlib_prefetch_buffer_header (p3, LOAD);
1598 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1599 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1602 /* speculatively enqueue b0 and b1 to the current next frame */
1603 to_next[0] = bi0 = from[0];
1604 to_next[1] = bi1 = from[1];
1608 n_left_to_next -= 2;
1610 b0 = vlib_get_buffer (vm, bi0);
1611 b1 = vlib_get_buffer (vm, bi1);
1613 ip0 = vlib_buffer_get_current (b0);
1614 udp0 = ip4_next_header (ip0);
1615 tcp0 = (tcp_header_t *) udp0;
1617 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1619 if (PREDICT_FALSE(ip0->ttl == 1))
1621 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1622 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1623 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1625 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1629 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1631 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1633 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1634 icmp0 = (icmp46_header_t *) udp0;
1636 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1637 rx_fib_index0, node, next0, thread_index,
1642 key0.ext_host_addr = ip0->src_address;
1643 key0.ext_host_port = tcp0->src;
1644 key0.out_port = tcp0->dst;
1646 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1647 if (PREDICT_FALSE(!dm0))
1649 clib_warning("unknown dst address: %U",
1650 format_ip4_address, &ip0->dst_address);
1651 next0 = SNAT_OUT2IN_NEXT_DROP;
1652 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1656 snat_det_reverse(dm0, &ip0->dst_address,
1657 clib_net_to_host_u16(tcp0->dst), &new_addr0);
1659 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1660 if (PREDICT_FALSE(!ses0))
1662 clib_warning("no match src %U:%d dst %U:%d for user %U",
1663 format_ip4_address, &ip0->src_address,
1664 clib_net_to_host_u16 (tcp0->src),
1665 format_ip4_address, &ip0->dst_address,
1666 clib_net_to_host_u16 (tcp0->dst),
1667 format_ip4_address, &new_addr0);
1668 next0 = SNAT_OUT2IN_NEXT_DROP;
1669 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1672 new_port0 = ses0->in_port;
1674 old_addr0 = ip0->dst_address;
1675 ip0->dst_address = new_addr0;
1676 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1678 sum0 = ip0->checksum;
1679 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1681 dst_address /* changed member */);
1682 ip0->checksum = ip_csum_fold (sum0);
1684 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1686 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1687 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1688 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1689 snat_det_ses_close(dm0, ses0);
1691 old_port0 = tcp0->dst;
1692 tcp0->dst = new_port0;
1694 sum0 = tcp0->checksum;
1695 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1697 dst_address /* changed member */);
1699 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1700 ip4_header_t /* cheat */,
1701 length /* changed member */);
1702 tcp0->checksum = ip_csum_fold(sum0);
1706 old_port0 = udp0->dst_port;
1707 udp0->dst_port = new_port0;
1713 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1714 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1716 snat_out2in_trace_t *t =
1717 vlib_add_trace (vm, node, b0, sizeof (*t));
1718 t->sw_if_index = sw_if_index0;
1719 t->next_index = next0;
1720 t->session_index = ~0;
1722 t->session_index = ses0 - dm0->sessions;
1725 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1727 b1 = vlib_get_buffer (vm, bi1);
1729 ip1 = vlib_buffer_get_current (b1);
1730 udp1 = ip4_next_header (ip1);
1731 tcp1 = (tcp_header_t *) udp1;
1733 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1735 if (PREDICT_FALSE(ip1->ttl == 1))
1737 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1738 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1739 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1741 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1745 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1747 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1749 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1750 icmp1 = (icmp46_header_t *) udp1;
1752 next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1753 rx_fib_index1, node, next1, thread_index,
1758 key1.ext_host_addr = ip1->src_address;
1759 key1.ext_host_port = tcp1->src;
1760 key1.out_port = tcp1->dst;
1762 dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1763 if (PREDICT_FALSE(!dm1))
1765 clib_warning("unknown dst address: %U",
1766 format_ip4_address, &ip1->dst_address);
1767 next1 = SNAT_OUT2IN_NEXT_DROP;
1768 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1772 snat_det_reverse(dm1, &ip1->dst_address,
1773 clib_net_to_host_u16(tcp1->dst), &new_addr1);
1775 ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1776 if (PREDICT_FALSE(!ses1))
1778 clib_warning("no match src %U:%d dst %U:%d for user %U",
1779 format_ip4_address, &ip1->src_address,
1780 clib_net_to_host_u16 (tcp1->src),
1781 format_ip4_address, &ip1->dst_address,
1782 clib_net_to_host_u16 (tcp1->dst),
1783 format_ip4_address, &new_addr1);
1784 next1 = SNAT_OUT2IN_NEXT_DROP;
1785 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1788 new_port1 = ses1->in_port;
1790 old_addr1 = ip1->dst_address;
1791 ip1->dst_address = new_addr1;
1792 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1794 sum1 = ip1->checksum;
1795 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1797 dst_address /* changed member */);
1798 ip1->checksum = ip_csum_fold (sum1);
1800 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1802 if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1803 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1804 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1805 snat_det_ses_close(dm1, ses1);
1807 old_port1 = tcp1->dst;
1808 tcp1->dst = new_port1;
1810 sum1 = tcp1->checksum;
1811 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1813 dst_address /* changed member */);
1815 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1816 ip4_header_t /* cheat */,
1817 length /* changed member */);
1818 tcp1->checksum = ip_csum_fold(sum1);
1822 old_port1 = udp1->dst_port;
1823 udp1->dst_port = new_port1;
1829 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1830 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1832 snat_out2in_trace_t *t =
1833 vlib_add_trace (vm, node, b1, sizeof (*t));
1834 t->sw_if_index = sw_if_index1;
1835 t->next_index = next1;
1836 t->session_index = ~0;
1838 t->session_index = ses1 - dm1->sessions;
1841 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1843 /* verify speculative enqueues, maybe switch current next frame */
1844 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1845 to_next, n_left_to_next,
1846 bi0, bi1, next0, next1);
1849 while (n_left_from > 0 && n_left_to_next > 0)
1853 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1857 ip4_address_t new_addr0, old_addr0;
1858 u16 new_port0, old_port0;
1859 udp_header_t * udp0;
1860 tcp_header_t * tcp0;
1862 snat_det_out_key_t key0;
1863 snat_det_map_t * dm0;
1864 snat_det_session_t * ses0 = 0;
1866 icmp46_header_t * icmp0;
1868 /* speculatively enqueue b0 to the current next frame */
1874 n_left_to_next -= 1;
1876 b0 = vlib_get_buffer (vm, bi0);
1878 ip0 = vlib_buffer_get_current (b0);
1879 udp0 = ip4_next_header (ip0);
1880 tcp0 = (tcp_header_t *) udp0;
1882 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1884 if (PREDICT_FALSE(ip0->ttl == 1))
1886 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1887 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1888 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1890 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1894 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1896 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1898 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1899 icmp0 = (icmp46_header_t *) udp0;
1901 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1902 rx_fib_index0, node, next0, thread_index,
1907 key0.ext_host_addr = ip0->src_address;
1908 key0.ext_host_port = tcp0->src;
1909 key0.out_port = tcp0->dst;
1911 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1912 if (PREDICT_FALSE(!dm0))
1914 clib_warning("unknown dst address: %U",
1915 format_ip4_address, &ip0->dst_address);
1916 next0 = SNAT_OUT2IN_NEXT_DROP;
1917 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1921 snat_det_reverse(dm0, &ip0->dst_address,
1922 clib_net_to_host_u16(tcp0->dst), &new_addr0);
1924 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1925 if (PREDICT_FALSE(!ses0))
1927 clib_warning("no match src %U:%d dst %U:%d for user %U",
1928 format_ip4_address, &ip0->src_address,
1929 clib_net_to_host_u16 (tcp0->src),
1930 format_ip4_address, &ip0->dst_address,
1931 clib_net_to_host_u16 (tcp0->dst),
1932 format_ip4_address, &new_addr0);
1933 next0 = SNAT_OUT2IN_NEXT_DROP;
1934 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1937 new_port0 = ses0->in_port;
1939 old_addr0 = ip0->dst_address;
1940 ip0->dst_address = new_addr0;
1941 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1943 sum0 = ip0->checksum;
1944 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1946 dst_address /* changed member */);
1947 ip0->checksum = ip_csum_fold (sum0);
1949 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1951 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1952 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1953 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1954 snat_det_ses_close(dm0, ses0);
1956 old_port0 = tcp0->dst;
1957 tcp0->dst = new_port0;
1959 sum0 = tcp0->checksum;
1960 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1962 dst_address /* changed member */);
1964 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1965 ip4_header_t /* cheat */,
1966 length /* changed member */);
1967 tcp0->checksum = ip_csum_fold(sum0);
1971 old_port0 = udp0->dst_port;
1972 udp0->dst_port = new_port0;
1978 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1979 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1981 snat_out2in_trace_t *t =
1982 vlib_add_trace (vm, node, b0, sizeof (*t));
1983 t->sw_if_index = sw_if_index0;
1984 t->next_index = next0;
1985 t->session_index = ~0;
1987 t->session_index = ses0 - dm0->sessions;
1990 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1992 /* verify speculative enqueue, maybe switch current next frame */
1993 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1994 to_next, n_left_to_next,
1998 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2001 vlib_node_increment_counter (vm, snat_det_out2in_node.index,
2002 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2004 return frame->n_vectors;
2007 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
2008 .function = snat_det_out2in_node_fn,
2009 .name = "nat44-det-out2in",
2010 .vector_size = sizeof (u32),
2011 .format_trace = format_snat_out2in_trace,
2012 .type = VLIB_NODE_TYPE_INTERNAL,
2014 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2015 .error_strings = snat_out2in_error_strings,
2017 .runtime_data_bytes = sizeof (snat_runtime_t),
2019 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2021 /* edit / add dispositions here */
2023 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2024 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2025 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2028 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
2031 * Get address and port values to be used for ICMP packet translation
2032 * and create session if needed
2034 * @param[in,out] sm NAT main
2035 * @param[in,out] node NAT node runtime
2036 * @param[in] thread_index thread index
2037 * @param[in,out] b0 buffer containing packet to be translated
2038 * @param[out] p_proto protocol used for matching
2039 * @param[out] p_value address and port after NAT translation
2040 * @param[out] p_dont_translate if packet should not be translated
2041 * @param d optional parameter
2042 * @param e optional parameter
2044 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
2045 u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
2046 snat_session_key_t *p_value,
2047 u8 *p_dont_translate, void *d, void *e)
2050 icmp46_header_t *icmp0;
2053 snat_det_out_key_t key0;
2054 u8 dont_translate = 0;
2056 icmp_echo_header_t *echo0, *inner_echo0 = 0;
2057 ip4_header_t *inner_ip0;
2058 void *l4_header = 0;
2059 icmp46_header_t *inner_icmp0;
2060 snat_det_map_t * dm0 = 0;
2061 ip4_address_t new_addr0 = {{0}};
2062 snat_det_session_t * ses0 = 0;
2063 ip4_address_t out_addr;
2065 ip0 = vlib_buffer_get_current (b0);
2066 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2067 echo0 = (icmp_echo_header_t *)(icmp0+1);
2068 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2070 if (!icmp_is_error_message (icmp0))
2072 protocol = SNAT_PROTOCOL_ICMP;
2073 key0.ext_host_addr = ip0->src_address;
2074 key0.ext_host_port = 0;
2075 key0.out_port = echo0->identifier;
2076 out_addr = ip0->dst_address;
2080 inner_ip0 = (ip4_header_t *)(echo0+1);
2081 l4_header = ip4_next_header (inner_ip0);
2082 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
2083 key0.ext_host_addr = inner_ip0->dst_address;
2084 out_addr = inner_ip0->src_address;
2087 case SNAT_PROTOCOL_ICMP:
2088 inner_icmp0 = (icmp46_header_t*)l4_header;
2089 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2090 key0.ext_host_port = 0;
2091 key0.out_port = inner_echo0->identifier;
2093 case SNAT_PROTOCOL_UDP:
2094 case SNAT_PROTOCOL_TCP:
2095 key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2096 key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
2099 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
2100 next0 = SNAT_OUT2IN_NEXT_DROP;
2105 dm0 = snat_det_map_by_out(sm, &out_addr);
2106 if (PREDICT_FALSE(!dm0))
2108 /* Don't NAT packet aimed at the intfc address */
2109 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2110 ip0->dst_address.as_u32)))
2115 clib_warning("unknown dst address: %U",
2116 format_ip4_address, &ip0->dst_address);
2120 snat_det_reverse(dm0, &ip0->dst_address,
2121 clib_net_to_host_u16(key0.out_port), &new_addr0);
2123 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2124 if (PREDICT_FALSE(!ses0))
2126 /* Don't NAT packet aimed at the intfc address */
2127 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
2128 ip0->dst_address.as_u32)))
2133 clib_warning("no match src %U:%d dst %U:%d for user %U",
2134 format_ip4_address, &key0.ext_host_addr,
2135 clib_net_to_host_u16 (key0.ext_host_port),
2136 format_ip4_address, &out_addr,
2137 clib_net_to_host_u16 (key0.out_port),
2138 format_ip4_address, &new_addr0);
2139 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2140 next0 = SNAT_OUT2IN_NEXT_DROP;
2144 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
2145 !icmp_is_error_message (icmp0)))
2147 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2148 next0 = SNAT_OUT2IN_NEXT_DROP;
2155 *p_proto = protocol;
2158 p_value->addr = new_addr0;
2159 p_value->fib_index = sm->inside_fib_index;
2160 p_value->port = ses0->in_port;
2162 *p_dont_translate = dont_translate;
2164 *(snat_det_session_t**)d = ses0;
2166 *(snat_det_map_t**)e = dm0;
2170 /**********************/
2171 /*** worker handoff ***/
2172 /**********************/
2174 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
2175 vlib_node_runtime_t * node,
2176 vlib_frame_t * frame)
2178 snat_main_t *sm = &snat_main;
2179 vlib_thread_main_t *tm = vlib_get_thread_main ();
2180 u32 n_left_from, *from, *to_next = 0;
2181 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
2182 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
2184 vlib_frame_queue_elt_t *hf = 0;
2185 vlib_frame_t *f = 0;
2187 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
2188 u32 next_worker_index = 0;
2189 u32 current_worker_index = ~0;
2190 u32 thread_index = vlib_get_thread_index ();
2192 ASSERT (vec_len (sm->workers));
2194 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
2196 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
2198 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
2199 sm->first_worker_index + sm->num_workers - 1,
2200 (vlib_frame_queue_t *) (~0));
2203 from = vlib_frame_vector_args (frame);
2204 n_left_from = frame->n_vectors;
2206 while (n_left_from > 0)
2219 b0 = vlib_get_buffer (vm, bi0);
2221 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2222 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2224 ip0 = vlib_buffer_get_current (b0);
2226 next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
2228 if (PREDICT_FALSE (next_worker_index != thread_index))
2232 if (next_worker_index != current_worker_index)
2235 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2237 hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
2239 handoff_queue_elt_by_worker_index);
2241 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
2242 to_next_worker = &hf->buffer_index[hf->n_vectors];
2243 current_worker_index = next_worker_index;
2246 /* enqueue to correct worker thread */
2247 to_next_worker[0] = bi0;
2249 n_left_to_next_worker--;
2251 if (n_left_to_next_worker == 0)
2253 hf->n_vectors = VLIB_FRAME_SIZE;
2254 vlib_put_frame_queue_elt (hf);
2255 current_worker_index = ~0;
2256 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
2263 /* if this is 1st frame */
2266 f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
2267 to_next = vlib_frame_vector_args (f);
2275 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
2276 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2278 snat_out2in_worker_handoff_trace_t *t =
2279 vlib_add_trace (vm, node, b0, sizeof (*t));
2280 t->next_worker_index = next_worker_index;
2281 t->do_handoff = do_handoff;
2286 vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
2289 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
2291 /* Ship frames to the worker nodes */
2292 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
2294 if (handoff_queue_elt_by_worker_index[i])
2296 hf = handoff_queue_elt_by_worker_index[i];
2298 * It works better to let the handoff node
2299 * rate-adapt, always ship the handoff queue element.
2301 if (1 || hf->n_vectors == hf->last_n_vectors)
2303 vlib_put_frame_queue_elt (hf);
2304 handoff_queue_elt_by_worker_index[i] = 0;
2307 hf->last_n_vectors = hf->n_vectors;
2309 congested_handoff_queue_by_worker_index[i] =
2310 (vlib_frame_queue_t *) (~0);
2313 current_worker_index = ~0;
2314 return frame->n_vectors;
2317 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
2318 .function = snat_out2in_worker_handoff_fn,
2319 .name = "nat44-out2in-worker-handoff",
2320 .vector_size = sizeof (u32),
2321 .format_trace = format_snat_out2in_worker_handoff_trace,
2322 .type = VLIB_NODE_TYPE_INTERNAL,
2331 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
2334 snat_out2in_fast_node_fn (vlib_main_t * vm,
2335 vlib_node_runtime_t * node,
2336 vlib_frame_t * frame)
2338 u32 n_left_from, * from, * to_next;
2339 snat_out2in_next_t next_index;
2340 u32 pkts_processed = 0;
2341 snat_main_t * sm = &snat_main;
2343 from = vlib_frame_vector_args (frame);
2344 n_left_from = frame->n_vectors;
2345 next_index = node->cached_next_index;
2347 while (n_left_from > 0)
2351 vlib_get_next_frame (vm, node, next_index,
2352 to_next, n_left_to_next);
2354 while (n_left_from > 0 && n_left_to_next > 0)
2358 u32 next0 = SNAT_OUT2IN_NEXT_DROP;
2362 u32 new_addr0, old_addr0;
2363 u16 new_port0, old_port0;
2364 udp_header_t * udp0;
2365 tcp_header_t * tcp0;
2366 icmp46_header_t * icmp0;
2367 snat_session_key_t key0, sm0;
2371 /* speculatively enqueue b0 to the current next frame */
2377 n_left_to_next -= 1;
2379 b0 = vlib_get_buffer (vm, bi0);
2381 ip0 = vlib_buffer_get_current (b0);
2382 udp0 = ip4_next_header (ip0);
2383 tcp0 = (tcp_header_t *) udp0;
2384 icmp0 = (icmp46_header_t *) udp0;
2386 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2387 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2389 vnet_feature_next (sw_if_index0, &next0, b0);
2391 if (PREDICT_FALSE(ip0->ttl == 1))
2393 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2394 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2395 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2397 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2401 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2403 if (PREDICT_FALSE (proto0 == ~0))
2406 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2408 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2409 rx_fib_index0, node, next0, ~0, 0, 0);
2413 key0.addr = ip0->dst_address;
2414 key0.port = udp0->dst_port;
2415 key0.fib_index = rx_fib_index0;
2417 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2419 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2423 new_addr0 = sm0.addr.as_u32;
2424 new_port0 = sm0.port;
2425 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2426 old_addr0 = ip0->dst_address.as_u32;
2427 ip0->dst_address.as_u32 = new_addr0;
2429 sum0 = ip0->checksum;
2430 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2432 dst_address /* changed member */);
2433 ip0->checksum = ip_csum_fold (sum0);
2435 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2437 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2439 old_port0 = tcp0->dst_port;
2440 tcp0->dst_port = new_port0;
2442 sum0 = tcp0->checksum;
2443 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2445 dst_address /* changed member */);
2447 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2448 ip4_header_t /* cheat */,
2449 length /* changed member */);
2450 tcp0->checksum = ip_csum_fold(sum0);
2454 old_port0 = udp0->dst_port;
2455 udp0->dst_port = new_port0;
2461 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2463 sum0 = tcp0->checksum;
2464 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2466 dst_address /* changed member */);
2468 tcp0->checksum = ip_csum_fold(sum0);
2474 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2475 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2477 snat_out2in_trace_t *t =
2478 vlib_add_trace (vm, node, b0, sizeof (*t));
2479 t->sw_if_index = sw_if_index0;
2480 t->next_index = next0;
2483 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2485 /* verify speculative enqueue, maybe switch current next frame */
2486 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2487 to_next, n_left_to_next,
2491 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2494 vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2495 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2497 return frame->n_vectors;
2500 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2501 .function = snat_out2in_fast_node_fn,
2502 .name = "nat44-out2in-fast",
2503 .vector_size = sizeof (u32),
2504 .format_trace = format_snat_out2in_fast_trace,
2505 .type = VLIB_NODE_TYPE_INTERNAL,
2507 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2508 .error_strings = snat_out2in_error_strings,
2510 .runtime_data_bytes = sizeof (snat_runtime_t),
2512 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2514 /* edit / add dispositions here */
2516 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2517 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2518 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2521 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);