2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
25 #include <snat/snat.h>
26 #include <snat/snat_ipfix_logging.h>
27 #include <snat/snat_det.h>
29 #include <vppinfra/hash.h>
30 #include <vppinfra/error.h>
31 #include <vppinfra/elog.h>
37 } snat_out2in_trace_t;
40 u32 next_worker_index;
42 } snat_out2in_worker_handoff_trace_t;
44 /* packet trace format function */
45 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
47 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
48 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
49 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
51 s = format (s, "SNAT_OUT2IN: sw_if_index %d, next index %d, session index %d",
52 t->sw_if_index, t->next_index, t->session_index);
56 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
58 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
59 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
60 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
62 s = format (s, "SNAT_OUT2IN_FAST: sw_if_index %d, next index %d",
63 t->sw_if_index, t->next_index);
67 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
69 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
70 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
71 snat_out2in_worker_handoff_trace_t * t =
72 va_arg (*args, snat_out2in_worker_handoff_trace_t *);
75 m = t->do_handoff ? "next worker" : "same worker";
76 s = format (s, "SNAT_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
81 vlib_node_registration_t snat_out2in_node;
82 vlib_node_registration_t snat_out2in_fast_node;
83 vlib_node_registration_t snat_out2in_worker_handoff_node;
84 vlib_node_registration_t snat_det_out2in_node;
86 #define foreach_snat_out2in_error \
87 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
88 _(OUT2IN_PACKETS, "Good out2in packets processed") \
89 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
90 _(NO_TRANSLATION, "No translation")
93 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
94 foreach_snat_out2in_error
97 } snat_out2in_error_t;
99 static char * snat_out2in_error_strings[] = {
100 #define _(sym,string) string,
101 foreach_snat_out2in_error
106 SNAT_OUT2IN_NEXT_DROP,
107 SNAT_OUT2IN_NEXT_LOOKUP,
108 SNAT_OUT2IN_NEXT_ICMP_ERROR,
110 } snat_out2in_next_t;
113 * @brief Create session for static mapping.
115 * Create NAT session initiated by host from external network with static
118 * @param sm SNAT main.
119 * @param b0 Vlib buffer.
120 * @param in2out In2out SNAT session key.
121 * @param out2in Out2in SNAT session key.
122 * @param node Vlib node.
124 * @returns SNAT session if successfully created otherwise 0.
126 static inline snat_session_t *
127 create_session_for_static_mapping (snat_main_t *sm,
129 snat_session_key_t in2out,
130 snat_session_key_t out2in,
131 vlib_node_runtime_t * node,
135 snat_user_key_t user_key;
137 clib_bihash_kv_8_8_t kv0, value0;
138 dlist_elt_t * per_user_translation_list_elt;
139 dlist_elt_t * per_user_list_head_elt;
141 user_key.addr = in2out.addr;
142 user_key.fib_index = in2out.fib_index;
143 kv0.key = user_key.as_u64;
145 /* Ever heard of the "user" = inside ip4 address before? */
146 if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0))
148 /* no, make a new one */
149 pool_get (sm->per_thread_data[thread_index].users, u);
150 memset (u, 0, sizeof (*u));
151 u->addr = in2out.addr;
152 u->fib_index = in2out.fib_index;
154 pool_get (sm->per_thread_data[thread_index].list_pool,
155 per_user_list_head_elt);
157 u->sessions_per_user_list_head_index = per_user_list_head_elt -
158 sm->per_thread_data[thread_index].list_pool;
160 clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
161 u->sessions_per_user_list_head_index);
163 kv0.value = u - sm->per_thread_data[thread_index].users;
166 clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */);
168 /* add non-traslated packets worker lookup */
169 kv0.value = thread_index;
170 clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1);
174 u = pool_elt_at_index (sm->per_thread_data[thread_index].users,
178 pool_get (sm->per_thread_data[thread_index].sessions, s);
179 memset (s, 0, sizeof (*s));
181 s->outside_address_index = ~0;
182 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
183 u->nstaticsessions++;
185 /* Create list elts */
186 pool_get (sm->per_thread_data[thread_index].list_pool,
187 per_user_translation_list_elt);
188 clib_dlist_init (sm->per_thread_data[thread_index].list_pool,
189 per_user_translation_list_elt -
190 sm->per_thread_data[thread_index].list_pool);
192 per_user_translation_list_elt->value =
193 s - sm->per_thread_data[thread_index].sessions;
195 per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool;
196 s->per_user_list_head_index = u->sessions_per_user_list_head_index;
198 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
199 s->per_user_list_head_index,
200 per_user_translation_list_elt -
201 sm->per_thread_data[thread_index].list_pool);
205 s->in2out.protocol = out2in.protocol;
207 /* Add to translation hashes */
208 kv0.key = s->in2out.as_u64;
209 kv0.value = s - sm->per_thread_data[thread_index].sessions;
210 if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */))
211 clib_warning ("in2out key add failed");
213 kv0.key = s->out2in.as_u64;
214 kv0.value = s - sm->per_thread_data[thread_index].sessions;
216 if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */))
217 clib_warning ("out2in key add failed");
220 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
221 s->out2in.addr.as_u32,
225 s->in2out.fib_index);
230 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
231 snat_session_key_t *p_key0)
233 icmp46_header_t *icmp0;
234 snat_session_key_t key0;
235 icmp_echo_header_t *echo0, *inner_echo0 = 0;
236 ip4_header_t *inner_ip0;
238 icmp46_header_t *inner_icmp0;
240 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
241 echo0 = (icmp_echo_header_t *)(icmp0+1);
243 if (!icmp_is_error_message (icmp0))
245 key0.protocol = SNAT_PROTOCOL_ICMP;
246 key0.addr = ip0->dst_address;
247 key0.port = echo0->identifier;
251 inner_ip0 = (ip4_header_t *)(echo0+1);
252 l4_header = ip4_next_header (inner_ip0);
253 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
254 key0.addr = inner_ip0->src_address;
255 switch (key0.protocol)
257 case SNAT_PROTOCOL_ICMP:
258 inner_icmp0 = (icmp46_header_t*)l4_header;
259 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
260 key0.port = inner_echo0->identifier;
262 case SNAT_PROTOCOL_UDP:
263 case SNAT_PROTOCOL_TCP:
264 key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
267 return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
271 return -1; /* success */
275 * Get address and port values to be used for packet SNAT translation
276 * and create session if needed
278 * @param[in,out] sm SNAT main
279 * @param[in,out] node SNAT node runtime
280 * @param[in] thread_index thread index
281 * @param[in,out] b0 buffer containing packet to be translated
282 * @param[out] p_proto protocol used for matching
283 * @param[out] p_value address and port after NAT translation
284 * @param[out] p_dont_translate if packet should not be translated
285 * @param d optional parameter
286 * @param e optional parameter
288 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
289 u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
290 snat_session_key_t *p_value,
291 u8 *p_dont_translate, void *d, void *e)
294 icmp46_header_t *icmp0;
297 snat_session_key_t key0;
298 snat_session_key_t sm0;
299 snat_session_t *s0 = 0;
300 u8 dont_translate = 0;
301 clib_bihash_kv_8_8_t kv0, value0;
306 ip0 = vlib_buffer_get_current (b0);
307 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
308 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
309 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
313 err = icmp_get_key (ip0, &key0);
316 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
317 next0 = SNAT_OUT2IN_NEXT_DROP;
320 key0.fib_index = rx_fib_index0;
322 kv0.key = key0.as_u64;
324 if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
326 /* Try to match static mapping by external address and port,
327 destination address and port in packet */
328 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
330 /* Don't NAT packet aimed at the intfc address */
331 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
332 ip0->dst_address.as_u32)))
337 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
338 next0 = SNAT_OUT2IN_NEXT_DROP;
342 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
343 (icmp0->type != ICMP4_echo_request || !is_addr_only)))
345 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
346 next0 = SNAT_OUT2IN_NEXT_DROP;
350 /* Create session initiated by host from external network */
351 s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
356 next0 = SNAT_OUT2IN_NEXT_DROP;
362 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
363 icmp0->type != ICMP4_echo_request &&
364 !icmp_is_error_message (icmp0)))
366 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
367 next0 = SNAT_OUT2IN_NEXT_DROP;
371 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
376 *p_proto = key0.protocol;
378 *p_value = s0->in2out;
379 *p_dont_translate = dont_translate;
381 *(snat_session_t**)d = s0;
386 * Get address and port values to be used for packet SNAT translation
388 * @param[in] sm SNAT main
389 * @param[in,out] node SNAT node runtime
390 * @param[in] thread_index thread index
391 * @param[in,out] b0 buffer containing packet to be translated
392 * @param[out] p_proto protocol used for matching
393 * @param[out] p_value address and port after NAT translation
394 * @param[out] p_dont_translate if packet should not be translated
395 * @param d optional parameter
396 * @param e optional parameter
398 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
399 u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
400 snat_session_key_t *p_value,
401 u8 *p_dont_translate, void *d, void *e)
404 icmp46_header_t *icmp0;
407 snat_session_key_t key0;
408 snat_session_key_t sm0;
409 u8 dont_translate = 0;
414 ip0 = vlib_buffer_get_current (b0);
415 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
416 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
417 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
419 err = icmp_get_key (ip0, &key0);
422 b0->error = node->errors[err];
423 next0 = SNAT_OUT2IN_NEXT_DROP;
426 key0.fib_index = rx_fib_index0;
428 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only))
430 /* Don't NAT packet aimed at the intfc address */
431 if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
436 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
437 next0 = SNAT_OUT2IN_NEXT_DROP;
441 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
442 (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
443 !icmp_is_error_message (icmp0)))
445 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
446 next0 = SNAT_OUT2IN_NEXT_DROP;
453 *p_proto = key0.protocol;
454 *p_dont_translate = dont_translate;
458 static inline u32 icmp_out2in (snat_main_t *sm,
461 icmp46_header_t * icmp0,
464 vlib_node_runtime_t * node,
470 snat_session_key_t sm0;
472 icmp_echo_header_t *echo0, *inner_echo0 = 0;
473 ip4_header_t *inner_ip0 = 0;
475 icmp46_header_t *inner_icmp0;
477 u32 new_addr0, old_addr0;
478 u16 old_id0, new_id0;
483 echo0 = (icmp_echo_header_t *)(icmp0+1);
485 next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0,
486 &protocol, &sm0, &dont_translate, d, e);
489 if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
492 sum0 = ip_incremental_checksum (0, icmp0,
493 ntohs(ip0->length) - ip4_header_bytes (ip0));
494 checksum0 = ~ip_csum_fold (sum0);
495 if (checksum0 != 0 && checksum0 != 0xffff)
497 next0 = SNAT_OUT2IN_NEXT_DROP;
501 old_addr0 = ip0->dst_address.as_u32;
502 new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
503 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
505 sum0 = ip0->checksum;
506 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
507 dst_address /* changed member */);
508 ip0->checksum = ip_csum_fold (sum0);
510 if (!icmp_is_error_message (icmp0))
513 if (PREDICT_FALSE(new_id0 != echo0->identifier))
515 old_id0 = echo0->identifier;
517 echo0->identifier = new_id0;
519 sum0 = icmp0->checksum;
520 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
521 identifier /* changed member */);
522 icmp0->checksum = ip_csum_fold (sum0);
527 inner_ip0 = (ip4_header_t *)(echo0+1);
528 l4_header = ip4_next_header (inner_ip0);
530 if (!ip4_header_checksum_is_valid (inner_ip0))
532 next0 = SNAT_OUT2IN_NEXT_DROP;
536 old_addr0 = inner_ip0->src_address.as_u32;
537 inner_ip0->src_address = sm0.addr;
538 new_addr0 = inner_ip0->src_address.as_u32;
540 sum0 = icmp0->checksum;
541 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
542 src_address /* changed member */);
543 icmp0->checksum = ip_csum_fold (sum0);
547 case SNAT_PROTOCOL_ICMP:
548 inner_icmp0 = (icmp46_header_t*)l4_header;
549 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
551 old_id0 = inner_echo0->identifier;
553 inner_echo0->identifier = new_id0;
555 sum0 = icmp0->checksum;
556 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
558 icmp0->checksum = ip_csum_fold (sum0);
560 case SNAT_PROTOCOL_UDP:
561 case SNAT_PROTOCOL_TCP:
562 old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
564 ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
566 sum0 = icmp0->checksum;
567 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
569 icmp0->checksum = ip_csum_fold (sum0);
581 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
584 icmp46_header_t * icmp0,
587 vlib_node_runtime_t * node,
590 snat_session_t ** p_s0)
592 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
593 next0, thread_index, p_s0, 0);
594 snat_session_t * s0 = *p_s0;
595 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
598 s0->last_heard = now;
600 s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
601 /* Per-user LRU list maintenance for dynamic translation */
602 if (!snat_is_session_static (s0))
604 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
606 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
607 s0->per_user_list_head_index,
615 snat_out2in_node_fn (vlib_main_t * vm,
616 vlib_node_runtime_t * node,
617 vlib_frame_t * frame)
619 u32 n_left_from, * from, * to_next;
620 snat_out2in_next_t next_index;
621 u32 pkts_processed = 0;
622 snat_main_t * sm = &snat_main;
623 f64 now = vlib_time_now (vm);
624 u32 thread_index = vlib_get_thread_index ();
626 from = vlib_frame_vector_args (frame);
627 n_left_from = frame->n_vectors;
628 next_index = node->cached_next_index;
630 while (n_left_from > 0)
634 vlib_get_next_frame (vm, node, next_index,
635 to_next, n_left_to_next);
637 while (n_left_from >= 4 && n_left_to_next >= 2)
640 vlib_buffer_t * b0, * b1;
641 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
642 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
643 u32 sw_if_index0, sw_if_index1;
644 ip4_header_t * ip0, *ip1;
645 ip_csum_t sum0, sum1;
646 u32 new_addr0, old_addr0;
647 u16 new_port0, old_port0;
648 u32 new_addr1, old_addr1;
649 u16 new_port1, old_port1;
650 udp_header_t * udp0, * udp1;
651 tcp_header_t * tcp0, * tcp1;
652 icmp46_header_t * icmp0, * icmp1;
653 snat_session_key_t key0, key1, sm0, sm1;
654 u32 rx_fib_index0, rx_fib_index1;
656 snat_session_t * s0 = 0, * s1 = 0;
657 clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
659 /* Prefetch next iteration. */
661 vlib_buffer_t * p2, * p3;
663 p2 = vlib_get_buffer (vm, from[2]);
664 p3 = vlib_get_buffer (vm, from[3]);
666 vlib_prefetch_buffer_header (p2, LOAD);
667 vlib_prefetch_buffer_header (p3, LOAD);
669 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
670 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
673 /* speculatively enqueue b0 and b1 to the current next frame */
674 to_next[0] = bi0 = from[0];
675 to_next[1] = bi1 = from[1];
681 b0 = vlib_get_buffer (vm, bi0);
682 b1 = vlib_get_buffer (vm, bi1);
684 ip0 = vlib_buffer_get_current (b0);
685 udp0 = ip4_next_header (ip0);
686 tcp0 = (tcp_header_t *) udp0;
687 icmp0 = (icmp46_header_t *) udp0;
689 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
690 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
693 if (PREDICT_FALSE(ip0->ttl == 1))
695 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
696 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
697 ICMP4_time_exceeded_ttl_exceeded_in_transit,
699 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
703 proto0 = ip_proto_to_snat_proto (ip0->protocol);
705 if (PREDICT_FALSE (proto0 == ~0))
708 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
710 next0 = icmp_out2in_slow_path
711 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
712 next0, now, thread_index, &s0);
716 key0.addr = ip0->dst_address;
717 key0.port = udp0->dst_port;
718 key0.protocol = proto0;
719 key0.fib_index = rx_fib_index0;
721 kv0.key = key0.as_u64;
723 if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
725 /* Try to match static mapping by external address and port,
726 destination address and port in packet */
727 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
729 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
731 * Send DHCP packets to the ipv4 stack, or we won't
732 * be able to use dhcp client on the outside interface
734 if (proto0 != SNAT_PROTOCOL_UDP
736 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
737 next0 = SNAT_OUT2IN_NEXT_DROP;
741 /* Create session initiated by host from external network */
742 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
746 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
747 next0 = SNAT_OUT2IN_NEXT_DROP;
752 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
755 old_addr0 = ip0->dst_address.as_u32;
756 ip0->dst_address = s0->in2out.addr;
757 new_addr0 = ip0->dst_address.as_u32;
758 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
760 sum0 = ip0->checksum;
761 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
763 dst_address /* changed member */);
764 ip0->checksum = ip_csum_fold (sum0);
766 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
768 old_port0 = tcp0->dst_port;
769 tcp0->dst_port = s0->in2out.port;
770 new_port0 = tcp0->dst_port;
772 sum0 = tcp0->checksum;
773 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
775 dst_address /* changed member */);
777 sum0 = ip_csum_update (sum0, old_port0, new_port0,
778 ip4_header_t /* cheat */,
779 length /* changed member */);
780 tcp0->checksum = ip_csum_fold(sum0);
784 old_port0 = udp0->dst_port;
785 udp0->dst_port = s0->in2out.port;
790 s0->last_heard = now;
792 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
793 /* Per-user LRU list maintenance for dynamic translation */
794 if (!snat_is_session_static (s0))
796 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
798 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
799 s0->per_user_list_head_index,
804 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
805 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
807 snat_out2in_trace_t *t =
808 vlib_add_trace (vm, node, b0, sizeof (*t));
809 t->sw_if_index = sw_if_index0;
810 t->next_index = next0;
811 t->session_index = ~0;
813 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
816 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
819 ip1 = vlib_buffer_get_current (b1);
820 udp1 = ip4_next_header (ip1);
821 tcp1 = (tcp_header_t *) udp1;
822 icmp1 = (icmp46_header_t *) udp1;
824 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
825 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
828 if (PREDICT_FALSE(ip1->ttl == 1))
830 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
831 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
832 ICMP4_time_exceeded_ttl_exceeded_in_transit,
834 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
838 proto1 = ip_proto_to_snat_proto (ip1->protocol);
840 if (PREDICT_FALSE (proto1 == ~0))
843 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
845 next1 = icmp_out2in_slow_path
846 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
847 next1, now, thread_index, &s1);
851 key1.addr = ip1->dst_address;
852 key1.port = udp1->dst_port;
853 key1.protocol = proto1;
854 key1.fib_index = rx_fib_index1;
856 kv1.key = key1.as_u64;
858 if (clib_bihash_search_8_8 (&sm->out2in, &kv1, &value1))
860 /* Try to match static mapping by external address and port,
861 destination address and port in packet */
862 if (snat_static_mapping_match(sm, key1, &sm1, 1, 0))
864 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
866 * Send DHCP packets to the ipv4 stack, or we won't
867 * be able to use dhcp client on the outside interface
869 if (proto1 != SNAT_PROTOCOL_UDP
871 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
872 next1 = SNAT_OUT2IN_NEXT_DROP;
876 /* Create session initiated by host from external network */
877 s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
881 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
882 next1 = SNAT_OUT2IN_NEXT_DROP;
887 s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
890 old_addr1 = ip1->dst_address.as_u32;
891 ip1->dst_address = s1->in2out.addr;
892 new_addr1 = ip1->dst_address.as_u32;
893 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
895 sum1 = ip1->checksum;
896 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
898 dst_address /* changed member */);
899 ip1->checksum = ip_csum_fold (sum1);
901 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
903 old_port1 = tcp1->dst_port;
904 tcp1->dst_port = s1->in2out.port;
905 new_port1 = tcp1->dst_port;
907 sum1 = tcp1->checksum;
908 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
910 dst_address /* changed member */);
912 sum1 = ip_csum_update (sum1, old_port1, new_port1,
913 ip4_header_t /* cheat */,
914 length /* changed member */);
915 tcp1->checksum = ip_csum_fold(sum1);
919 old_port1 = udp1->dst_port;
920 udp1->dst_port = s1->in2out.port;
925 s1->last_heard = now;
927 s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
928 /* Per-user LRU list maintenance for dynamic translation */
929 if (!snat_is_session_static (s1))
931 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
933 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
934 s1->per_user_list_head_index,
939 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
940 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
942 snat_out2in_trace_t *t =
943 vlib_add_trace (vm, node, b1, sizeof (*t));
944 t->sw_if_index = sw_if_index1;
945 t->next_index = next1;
946 t->session_index = ~0;
948 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
951 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
953 /* verify speculative enqueues, maybe switch current next frame */
954 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
955 to_next, n_left_to_next,
956 bi0, bi1, next0, next1);
959 while (n_left_from > 0 && n_left_to_next > 0)
963 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
967 u32 new_addr0, old_addr0;
968 u16 new_port0, old_port0;
971 icmp46_header_t * icmp0;
972 snat_session_key_t key0, sm0;
975 snat_session_t * s0 = 0;
976 clib_bihash_kv_8_8_t kv0, value0;
978 /* speculatively enqueue b0 to the current next frame */
986 b0 = vlib_get_buffer (vm, bi0);
988 ip0 = vlib_buffer_get_current (b0);
989 udp0 = ip4_next_header (ip0);
990 tcp0 = (tcp_header_t *) udp0;
991 icmp0 = (icmp46_header_t *) udp0;
993 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
994 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
997 proto0 = ip_proto_to_snat_proto (ip0->protocol);
999 if (PREDICT_FALSE (proto0 == ~0))
1002 if (PREDICT_FALSE(ip0->ttl == 1))
1004 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1005 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1006 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1008 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1012 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1014 next0 = icmp_out2in_slow_path
1015 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1016 next0, now, thread_index, &s0);
1020 key0.addr = ip0->dst_address;
1021 key0.port = udp0->dst_port;
1022 key0.protocol = proto0;
1023 key0.fib_index = rx_fib_index0;
1025 kv0.key = key0.as_u64;
1027 if (clib_bihash_search_8_8 (&sm->out2in, &kv0, &value0))
1029 /* Try to match static mapping by external address and port,
1030 destination address and port in packet */
1031 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
1033 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1035 * Send DHCP packets to the ipv4 stack, or we won't
1036 * be able to use dhcp client on the outside interface
1038 if (proto0 != SNAT_PROTOCOL_UDP
1040 != clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client)))
1042 next0 = SNAT_OUT2IN_NEXT_DROP;
1046 /* Create session initiated by host from external network */
1047 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1051 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1052 next0 = SNAT_OUT2IN_NEXT_DROP;
1057 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1060 old_addr0 = ip0->dst_address.as_u32;
1061 ip0->dst_address = s0->in2out.addr;
1062 new_addr0 = ip0->dst_address.as_u32;
1063 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1065 sum0 = ip0->checksum;
1066 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1068 dst_address /* changed member */);
1069 ip0->checksum = ip_csum_fold (sum0);
1071 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1073 old_port0 = tcp0->dst_port;
1074 tcp0->dst_port = s0->in2out.port;
1075 new_port0 = tcp0->dst_port;
1077 sum0 = tcp0->checksum;
1078 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1080 dst_address /* changed member */);
1082 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1083 ip4_header_t /* cheat */,
1084 length /* changed member */);
1085 tcp0->checksum = ip_csum_fold(sum0);
1089 old_port0 = udp0->dst_port;
1090 udp0->dst_port = s0->in2out.port;
1095 s0->last_heard = now;
1097 s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1098 /* Per-user LRU list maintenance for dynamic translation */
1099 if (!snat_is_session_static (s0))
1101 clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1102 s0->per_user_index);
1103 clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1104 s0->per_user_list_head_index,
1105 s0->per_user_index);
1109 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1110 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1112 snat_out2in_trace_t *t =
1113 vlib_add_trace (vm, node, b0, sizeof (*t));
1114 t->sw_if_index = sw_if_index0;
1115 t->next_index = next0;
1116 t->session_index = ~0;
1118 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1121 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1123 /* verify speculative enqueue, maybe switch current next frame */
1124 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1125 to_next, n_left_to_next,
1129 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1132 vlib_node_increment_counter (vm, snat_out2in_node.index,
1133 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1135 return frame->n_vectors;
1138 VLIB_REGISTER_NODE (snat_out2in_node) = {
1139 .function = snat_out2in_node_fn,
1140 .name = "snat-out2in",
1141 .vector_size = sizeof (u32),
1142 .format_trace = format_snat_out2in_trace,
1143 .type = VLIB_NODE_TYPE_INTERNAL,
1145 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1146 .error_strings = snat_out2in_error_strings,
1148 .runtime_data_bytes = sizeof (snat_runtime_t),
1150 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1152 /* edit / add dispositions here */
1154 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1155 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1156 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1159 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1161 /**************************/
1162 /*** deterministic mode ***/
1163 /**************************/
1165 snat_det_out2in_node_fn (vlib_main_t * vm,
1166 vlib_node_runtime_t * node,
1167 vlib_frame_t * frame)
1169 u32 n_left_from, * from, * to_next;
1170 snat_out2in_next_t next_index;
1171 u32 pkts_processed = 0;
1172 snat_main_t * sm = &snat_main;
1173 u32 thread_index = vlib_get_thread_index ();
1175 from = vlib_frame_vector_args (frame);
1176 n_left_from = frame->n_vectors;
1177 next_index = node->cached_next_index;
1179 while (n_left_from > 0)
1183 vlib_get_next_frame (vm, node, next_index,
1184 to_next, n_left_to_next);
1186 while (n_left_from >= 4 && n_left_to_next >= 2)
1189 vlib_buffer_t * b0, * b1;
1190 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1191 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
1192 u32 sw_if_index0, sw_if_index1;
1193 ip4_header_t * ip0, * ip1;
1194 ip_csum_t sum0, sum1;
1195 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
1196 u16 new_port0, old_port0, old_port1, new_port1;
1197 udp_header_t * udp0, * udp1;
1198 tcp_header_t * tcp0, * tcp1;
1200 snat_det_out_key_t key0, key1;
1201 snat_det_map_t * dm0, * dm1;
1202 snat_det_session_t * ses0 = 0, * ses1 = 0;
1203 u32 rx_fib_index0, rx_fib_index1;
1204 icmp46_header_t * icmp0, * icmp1;
1206 /* Prefetch next iteration. */
1208 vlib_buffer_t * p2, * p3;
1210 p2 = vlib_get_buffer (vm, from[2]);
1211 p3 = vlib_get_buffer (vm, from[3]);
1213 vlib_prefetch_buffer_header (p2, LOAD);
1214 vlib_prefetch_buffer_header (p3, LOAD);
1216 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1217 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1220 /* speculatively enqueue b0 and b1 to the current next frame */
1221 to_next[0] = bi0 = from[0];
1222 to_next[1] = bi1 = from[1];
1226 n_left_to_next -= 2;
1228 b0 = vlib_get_buffer (vm, bi0);
1229 b1 = vlib_get_buffer (vm, bi1);
1231 ip0 = vlib_buffer_get_current (b0);
1232 udp0 = ip4_next_header (ip0);
1233 tcp0 = (tcp_header_t *) udp0;
1235 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1237 if (PREDICT_FALSE(ip0->ttl == 1))
1239 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1240 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1241 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1243 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1247 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1249 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1251 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1252 icmp0 = (icmp46_header_t *) udp0;
1254 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1255 rx_fib_index0, node, next0, thread_index,
1260 key0.ext_host_addr = ip0->src_address;
1261 key0.ext_host_port = tcp0->src;
1262 key0.out_port = tcp0->dst;
1264 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1265 if (PREDICT_FALSE(!dm0))
1267 clib_warning("unknown dst address: %U",
1268 format_ip4_address, &ip0->dst_address);
1269 next0 = SNAT_OUT2IN_NEXT_DROP;
1270 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1274 snat_det_reverse(dm0, &ip0->dst_address,
1275 clib_net_to_host_u16(tcp0->dst), &new_addr0);
1277 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1278 if (PREDICT_FALSE(!ses0))
1280 clib_warning("no match src %U:%d dst %U:%d for user %U",
1281 format_ip4_address, &ip0->src_address,
1282 clib_net_to_host_u16 (tcp0->src),
1283 format_ip4_address, &ip0->dst_address,
1284 clib_net_to_host_u16 (tcp0->dst),
1285 format_ip4_address, &new_addr0);
1286 next0 = SNAT_OUT2IN_NEXT_DROP;
1287 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1290 new_port0 = ses0->in_port;
1292 old_addr0 = ip0->dst_address;
1293 ip0->dst_address = new_addr0;
1294 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1296 sum0 = ip0->checksum;
1297 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1299 dst_address /* changed member */);
1300 ip0->checksum = ip_csum_fold (sum0);
1302 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1304 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1305 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1306 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1307 snat_det_ses_close(dm0, ses0);
1309 old_port0 = tcp0->dst;
1310 tcp0->dst = new_port0;
1312 sum0 = tcp0->checksum;
1313 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1315 dst_address /* changed member */);
1317 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1318 ip4_header_t /* cheat */,
1319 length /* changed member */);
1320 tcp0->checksum = ip_csum_fold(sum0);
1324 old_port0 = udp0->dst_port;
1325 udp0->dst_port = new_port0;
1331 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1332 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1334 snat_out2in_trace_t *t =
1335 vlib_add_trace (vm, node, b0, sizeof (*t));
1336 t->sw_if_index = sw_if_index0;
1337 t->next_index = next0;
1338 t->session_index = ~0;
1340 t->session_index = ses0 - dm0->sessions;
1343 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1345 b1 = vlib_get_buffer (vm, bi1);
1347 ip1 = vlib_buffer_get_current (b1);
1348 udp1 = ip4_next_header (ip1);
1349 tcp1 = (tcp_header_t *) udp1;
1351 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1353 if (PREDICT_FALSE(ip1->ttl == 1))
1355 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1356 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1357 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1359 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1363 proto1 = ip_proto_to_snat_proto (ip1->protocol);
1365 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
1367 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
1368 icmp1 = (icmp46_header_t *) udp1;
1370 next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
1371 rx_fib_index1, node, next1, thread_index,
1376 key1.ext_host_addr = ip1->src_address;
1377 key1.ext_host_port = tcp1->src;
1378 key1.out_port = tcp1->dst;
1380 dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
1381 if (PREDICT_FALSE(!dm1))
1383 clib_warning("unknown dst address: %U",
1384 format_ip4_address, &ip1->dst_address);
1385 next1 = SNAT_OUT2IN_NEXT_DROP;
1386 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1390 snat_det_reverse(dm1, &ip1->dst_address,
1391 clib_net_to_host_u16(tcp1->dst), &new_addr1);
1393 ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
1394 if (PREDICT_FALSE(!ses1))
1396 clib_warning("no match src %U:%d dst %U:%d for user %U",
1397 format_ip4_address, &ip1->src_address,
1398 clib_net_to_host_u16 (tcp1->src),
1399 format_ip4_address, &ip1->dst_address,
1400 clib_net_to_host_u16 (tcp1->dst),
1401 format_ip4_address, &new_addr1);
1402 next1 = SNAT_OUT2IN_NEXT_DROP;
1403 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1406 new_port1 = ses1->in_port;
1408 old_addr1 = ip1->dst_address;
1409 ip1->dst_address = new_addr1;
1410 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1412 sum1 = ip1->checksum;
1413 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1415 dst_address /* changed member */);
1416 ip1->checksum = ip_csum_fold (sum1);
1418 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1420 if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
1421 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1422 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
1423 snat_det_ses_close(dm1, ses1);
1425 old_port1 = tcp1->dst;
1426 tcp1->dst = new_port1;
1428 sum1 = tcp1->checksum;
1429 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
1431 dst_address /* changed member */);
1433 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1434 ip4_header_t /* cheat */,
1435 length /* changed member */);
1436 tcp1->checksum = ip_csum_fold(sum1);
1440 old_port1 = udp1->dst_port;
1441 udp1->dst_port = new_port1;
1447 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1448 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1450 snat_out2in_trace_t *t =
1451 vlib_add_trace (vm, node, b1, sizeof (*t));
1452 t->sw_if_index = sw_if_index1;
1453 t->next_index = next1;
1454 t->session_index = ~0;
1456 t->session_index = ses1 - dm1->sessions;
1459 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1461 /* verify speculative enqueues, maybe switch current next frame */
1462 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1463 to_next, n_left_to_next,
1464 bi0, bi1, next0, next1);
1467 while (n_left_from > 0 && n_left_to_next > 0)
1471 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1475 ip4_address_t new_addr0, old_addr0;
1476 u16 new_port0, old_port0;
1477 udp_header_t * udp0;
1478 tcp_header_t * tcp0;
1480 snat_det_out_key_t key0;
1481 snat_det_map_t * dm0;
1482 snat_det_session_t * ses0 = 0;
1484 icmp46_header_t * icmp0;
1486 /* speculatively enqueue b0 to the current next frame */
1492 n_left_to_next -= 1;
1494 b0 = vlib_get_buffer (vm, bi0);
1496 ip0 = vlib_buffer_get_current (b0);
1497 udp0 = ip4_next_header (ip0);
1498 tcp0 = (tcp_header_t *) udp0;
1500 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1502 if (PREDICT_FALSE(ip0->ttl == 1))
1504 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1505 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1506 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1508 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1512 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1514 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
1516 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1517 icmp0 = (icmp46_header_t *) udp0;
1519 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
1520 rx_fib_index0, node, next0, thread_index,
1525 key0.ext_host_addr = ip0->src_address;
1526 key0.ext_host_port = tcp0->src;
1527 key0.out_port = tcp0->dst;
1529 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
1530 if (PREDICT_FALSE(!dm0))
1532 clib_warning("unknown dst address: %U",
1533 format_ip4_address, &ip0->dst_address);
1534 next0 = SNAT_OUT2IN_NEXT_DROP;
1535 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1539 snat_det_reverse(dm0, &ip0->dst_address,
1540 clib_net_to_host_u16(tcp0->dst), &new_addr0);
1542 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1543 if (PREDICT_FALSE(!ses0))
1545 clib_warning("no match src %U:%d dst %U:%d for user %U",
1546 format_ip4_address, &ip0->src_address,
1547 clib_net_to_host_u16 (tcp0->src),
1548 format_ip4_address, &ip0->dst_address,
1549 clib_net_to_host_u16 (tcp0->dst),
1550 format_ip4_address, &new_addr0);
1551 next0 = SNAT_OUT2IN_NEXT_DROP;
1552 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1555 new_port0 = ses0->in_port;
1557 old_addr0 = ip0->dst_address;
1558 ip0->dst_address = new_addr0;
1559 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
1561 sum0 = ip0->checksum;
1562 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1564 dst_address /* changed member */);
1565 ip0->checksum = ip_csum_fold (sum0);
1567 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1569 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
1570 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
1571 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
1572 snat_det_ses_close(dm0, ses0);
1574 old_port0 = tcp0->dst;
1575 tcp0->dst = new_port0;
1577 sum0 = tcp0->checksum;
1578 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
1580 dst_address /* changed member */);
1582 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1583 ip4_header_t /* cheat */,
1584 length /* changed member */);
1585 tcp0->checksum = ip_csum_fold(sum0);
1589 old_port0 = udp0->dst_port;
1590 udp0->dst_port = new_port0;
1596 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1597 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1599 snat_out2in_trace_t *t =
1600 vlib_add_trace (vm, node, b0, sizeof (*t));
1601 t->sw_if_index = sw_if_index0;
1602 t->next_index = next0;
1603 t->session_index = ~0;
1605 t->session_index = ses0 - dm0->sessions;
1608 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1610 /* verify speculative enqueue, maybe switch current next frame */
1611 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1612 to_next, n_left_to_next,
1616 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1619 vlib_node_increment_counter (vm, snat_det_out2in_node.index,
1620 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1622 return frame->n_vectors;
1625 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
1626 .function = snat_det_out2in_node_fn,
1627 .name = "snat-det-out2in",
1628 .vector_size = sizeof (u32),
1629 .format_trace = format_snat_out2in_trace,
1630 .type = VLIB_NODE_TYPE_INTERNAL,
1632 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1633 .error_strings = snat_out2in_error_strings,
1635 .runtime_data_bytes = sizeof (snat_runtime_t),
1637 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1639 /* edit / add dispositions here */
1641 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1642 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1643 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1646 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
1649 * Get address and port values to be used for packet SNAT translation
1650 * and create session if needed
1652 * @param[in,out] sm SNAT main
1653 * @param[in,out] node SNAT node runtime
1654 * @param[in] thread_index thread index
1655 * @param[in,out] b0 buffer containing packet to be translated
1656 * @param[out] p_proto protocol used for matching
1657 * @param[out] p_value address and port after NAT translation
1658 * @param[out] p_dont_translate if packet should not be translated
1659 * @param d optional parameter
1660 * @param e optional parameter
1662 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
1663 u32 thread_index, vlib_buffer_t *b0, u8 *p_proto,
1664 snat_session_key_t *p_value,
1665 u8 *p_dont_translate, void *d, void *e)
1668 icmp46_header_t *icmp0;
1671 snat_det_out_key_t key0;
1672 u8 dont_translate = 0;
1674 icmp_echo_header_t *echo0, *inner_echo0 = 0;
1675 ip4_header_t *inner_ip0;
1676 void *l4_header = 0;
1677 icmp46_header_t *inner_icmp0;
1678 snat_det_map_t * dm0 = 0;
1679 ip4_address_t new_addr0 = {{0}};
1680 snat_det_session_t * ses0 = 0;
1681 ip4_address_t out_addr;
1683 ip0 = vlib_buffer_get_current (b0);
1684 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1685 echo0 = (icmp_echo_header_t *)(icmp0+1);
1686 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1688 if (!icmp_is_error_message (icmp0))
1690 protocol = SNAT_PROTOCOL_ICMP;
1691 key0.ext_host_addr = ip0->src_address;
1692 key0.ext_host_port = 0;
1693 key0.out_port = echo0->identifier;
1694 out_addr = ip0->dst_address;
1698 inner_ip0 = (ip4_header_t *)(echo0+1);
1699 l4_header = ip4_next_header (inner_ip0);
1700 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
1701 key0.ext_host_addr = inner_ip0->dst_address;
1702 out_addr = inner_ip0->src_address;
1705 case SNAT_PROTOCOL_ICMP:
1706 inner_icmp0 = (icmp46_header_t*)l4_header;
1707 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1708 key0.ext_host_port = 0;
1709 key0.out_port = inner_echo0->identifier;
1711 case SNAT_PROTOCOL_UDP:
1712 case SNAT_PROTOCOL_TCP:
1713 key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1714 key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
1717 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1718 next0 = SNAT_OUT2IN_NEXT_DROP;
1723 dm0 = snat_det_map_by_out(sm, &out_addr);
1724 if (PREDICT_FALSE(!dm0))
1726 /* Don't NAT packet aimed at the intfc address */
1727 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
1728 ip0->dst_address.as_u32)))
1733 clib_warning("unknown dst address: %U",
1734 format_ip4_address, &ip0->dst_address);
1738 snat_det_reverse(dm0, &ip0->dst_address,
1739 clib_net_to_host_u16(key0.out_port), &new_addr0);
1741 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
1742 if (PREDICT_FALSE(!ses0))
1744 /* Don't NAT packet aimed at the intfc address */
1745 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
1746 ip0->dst_address.as_u32)))
1751 clib_warning("no match src %U:%d dst %U:%d for user %U",
1752 format_ip4_address, &key0.ext_host_addr,
1753 clib_net_to_host_u16 (key0.ext_host_port),
1754 format_ip4_address, &out_addr,
1755 clib_net_to_host_u16 (key0.out_port),
1756 format_ip4_address, &new_addr0);
1757 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1758 next0 = SNAT_OUT2IN_NEXT_DROP;
1762 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
1763 !icmp_is_error_message (icmp0)))
1765 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1766 next0 = SNAT_OUT2IN_NEXT_DROP;
1773 *p_proto = protocol;
1776 p_value->addr = new_addr0;
1777 p_value->fib_index = sm->inside_fib_index;
1778 p_value->port = ses0->in_port;
1780 *p_dont_translate = dont_translate;
1782 *(snat_det_session_t**)d = ses0;
1784 *(snat_det_map_t**)e = dm0;
1788 /**********************/
1789 /*** worker handoff ***/
1790 /**********************/
1792 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
1793 vlib_node_runtime_t * node,
1794 vlib_frame_t * frame)
1796 snat_main_t *sm = &snat_main;
1797 vlib_thread_main_t *tm = vlib_get_thread_main ();
1798 u32 n_left_from, *from, *to_next = 0;
1799 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
1800 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
1802 vlib_frame_queue_elt_t *hf = 0;
1803 vlib_frame_t *f = 0;
1805 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
1806 u32 next_worker_index = 0;
1807 u32 current_worker_index = ~0;
1808 u32 thread_index = vlib_get_thread_index ();
1810 ASSERT (vec_len (sm->workers));
1812 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
1814 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1816 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1817 sm->first_worker_index + sm->num_workers - 1,
1818 (vlib_frame_queue_t *) (~0));
1821 from = vlib_frame_vector_args (frame);
1822 n_left_from = frame->n_vectors;
1824 while (n_left_from > 0)
1837 b0 = vlib_get_buffer (vm, bi0);
1839 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1840 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
1842 ip0 = vlib_buffer_get_current (b0);
1844 next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
1846 if (PREDICT_FALSE (next_worker_index != thread_index))
1850 if (next_worker_index != current_worker_index)
1853 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1855 hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
1857 handoff_queue_elt_by_worker_index);
1859 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1860 to_next_worker = &hf->buffer_index[hf->n_vectors];
1861 current_worker_index = next_worker_index;
1864 /* enqueue to correct worker thread */
1865 to_next_worker[0] = bi0;
1867 n_left_to_next_worker--;
1869 if (n_left_to_next_worker == 0)
1871 hf->n_vectors = VLIB_FRAME_SIZE;
1872 vlib_put_frame_queue_elt (hf);
1873 current_worker_index = ~0;
1874 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1881 /* if this is 1st frame */
1884 f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
1885 to_next = vlib_frame_vector_args (f);
1893 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1894 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1896 snat_out2in_worker_handoff_trace_t *t =
1897 vlib_add_trace (vm, node, b0, sizeof (*t));
1898 t->next_worker_index = next_worker_index;
1899 t->do_handoff = do_handoff;
1904 vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
1907 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1909 /* Ship frames to the worker nodes */
1910 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1912 if (handoff_queue_elt_by_worker_index[i])
1914 hf = handoff_queue_elt_by_worker_index[i];
1916 * It works better to let the handoff node
1917 * rate-adapt, always ship the handoff queue element.
1919 if (1 || hf->n_vectors == hf->last_n_vectors)
1921 vlib_put_frame_queue_elt (hf);
1922 handoff_queue_elt_by_worker_index[i] = 0;
1925 hf->last_n_vectors = hf->n_vectors;
1927 congested_handoff_queue_by_worker_index[i] =
1928 (vlib_frame_queue_t *) (~0);
1931 current_worker_index = ~0;
1932 return frame->n_vectors;
1935 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
1936 .function = snat_out2in_worker_handoff_fn,
1937 .name = "snat-out2in-worker-handoff",
1938 .vector_size = sizeof (u32),
1939 .format_trace = format_snat_out2in_worker_handoff_trace,
1940 .type = VLIB_NODE_TYPE_INTERNAL,
1949 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
1952 snat_out2in_fast_node_fn (vlib_main_t * vm,
1953 vlib_node_runtime_t * node,
1954 vlib_frame_t * frame)
1956 u32 n_left_from, * from, * to_next;
1957 snat_out2in_next_t next_index;
1958 u32 pkts_processed = 0;
1959 snat_main_t * sm = &snat_main;
1961 from = vlib_frame_vector_args (frame);
1962 n_left_from = frame->n_vectors;
1963 next_index = node->cached_next_index;
1965 while (n_left_from > 0)
1969 vlib_get_next_frame (vm, node, next_index,
1970 to_next, n_left_to_next);
1972 while (n_left_from > 0 && n_left_to_next > 0)
1976 u32 next0 = SNAT_OUT2IN_NEXT_DROP;
1980 u32 new_addr0, old_addr0;
1981 u16 new_port0, old_port0;
1982 udp_header_t * udp0;
1983 tcp_header_t * tcp0;
1984 icmp46_header_t * icmp0;
1985 snat_session_key_t key0, sm0;
1989 /* speculatively enqueue b0 to the current next frame */
1995 n_left_to_next -= 1;
1997 b0 = vlib_get_buffer (vm, bi0);
1999 ip0 = vlib_buffer_get_current (b0);
2000 udp0 = ip4_next_header (ip0);
2001 tcp0 = (tcp_header_t *) udp0;
2002 icmp0 = (icmp46_header_t *) udp0;
2004 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2005 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2007 vnet_feature_next (sw_if_index0, &next0, b0);
2009 if (PREDICT_FALSE(ip0->ttl == 1))
2011 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2012 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2013 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2015 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2019 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2021 if (PREDICT_FALSE (proto0 == ~0))
2024 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2026 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2027 rx_fib_index0, node, next0, ~0, 0, 0);
2031 key0.addr = ip0->dst_address;
2032 key0.port = udp0->dst_port;
2033 key0.fib_index = rx_fib_index0;
2035 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0))
2037 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2041 new_addr0 = sm0.addr.as_u32;
2042 new_port0 = sm0.port;
2043 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2044 old_addr0 = ip0->dst_address.as_u32;
2045 ip0->dst_address.as_u32 = new_addr0;
2047 sum0 = ip0->checksum;
2048 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2050 dst_address /* changed member */);
2051 ip0->checksum = ip_csum_fold (sum0);
2053 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
2055 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2057 old_port0 = tcp0->dst_port;
2058 tcp0->dst_port = new_port0;
2060 sum0 = tcp0->checksum;
2061 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2063 dst_address /* changed member */);
2065 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2066 ip4_header_t /* cheat */,
2067 length /* changed member */);
2068 tcp0->checksum = ip_csum_fold(sum0);
2072 old_port0 = udp0->dst_port;
2073 udp0->dst_port = new_port0;
2079 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2081 sum0 = tcp0->checksum;
2082 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2084 dst_address /* changed member */);
2086 tcp0->checksum = ip_csum_fold(sum0);
2092 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2093 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2095 snat_out2in_trace_t *t =
2096 vlib_add_trace (vm, node, b0, sizeof (*t));
2097 t->sw_if_index = sw_if_index0;
2098 t->next_index = next0;
2101 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
2103 /* verify speculative enqueue, maybe switch current next frame */
2104 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2105 to_next, n_left_to_next,
2109 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2112 vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
2113 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2115 return frame->n_vectors;
2118 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
2119 .function = snat_out2in_fast_node_fn,
2120 .name = "snat-out2in-fast",
2121 .vector_size = sizeof (u32),
2122 .format_trace = format_snat_out2in_fast_trace,
2123 .type = VLIB_NODE_TYPE_INTERNAL,
2125 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2126 .error_strings = snat_out2in_error_strings,
2128 .runtime_data_bytes = sizeof (snat_runtime_t),
2130 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
2132 /* edit / add dispositions here */
2134 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2135 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
2136 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2139 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);