2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
26 #include <nat/nat_ipfix_logging.h>
27 #include <nat/nat_det.h>
28 #include <nat/nat_reass.h>
29 #include <nat/nat_inlines.h>
31 #include <vppinfra/hash.h>
32 #include <vppinfra/error.h>
33 #include <vppinfra/elog.h>
39 } snat_out2in_trace_t;
42 u32 next_worker_index;
44 } snat_out2in_worker_handoff_trace_t;
46 /* packet trace format function */
47 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
49 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
50 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
51 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
53 s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
54 t->sw_if_index, t->next_index, t->session_index);
58 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
60 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
61 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
62 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
64 s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
65 t->sw_if_index, t->next_index);
69 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
71 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
72 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
73 snat_out2in_worker_handoff_trace_t * t =
74 va_arg (*args, snat_out2in_worker_handoff_trace_t *);
77 m = t->do_handoff ? "next worker" : "same worker";
78 s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
87 } nat44_out2in_reass_trace_t;
89 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
91 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
92 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
93 nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
95 s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
96 t->sw_if_index, t->next_index,
97 t->cached ? "cached" : "translated");
102 vlib_node_registration_t snat_out2in_node;
103 vlib_node_registration_t snat_out2in_fast_node;
104 vlib_node_registration_t snat_out2in_worker_handoff_node;
105 vlib_node_registration_t snat_det_out2in_node;
106 vlib_node_registration_t nat44_out2in_reass_node;
107 vlib_node_registration_t nat44_ed_out2in_node;
108 vlib_node_registration_t nat44_ed_out2in_slowpath_node;
110 #define foreach_snat_out2in_error \
111 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
112 _(OUT2IN_PACKETS, "Good out2in packets processed") \
113 _(OUT_OF_PORTS, "Out of ports") \
114 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
115 _(NO_TRANSLATION, "No translation") \
116 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
117 _(DROP_FRAGMENT, "Drop fragment") \
118 _(MAX_REASS, "Maximum reassemblies exceeded") \
119 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
120 _(FQ_CONGESTED, "Handoff frame queue congested")
123 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
124 foreach_snat_out2in_error
127 } snat_out2in_error_t;
129 static char * snat_out2in_error_strings[] = {
130 #define _(sym,string) string,
131 foreach_snat_out2in_error
136 SNAT_OUT2IN_NEXT_DROP,
137 SNAT_OUT2IN_NEXT_LOOKUP,
138 SNAT_OUT2IN_NEXT_ICMP_ERROR,
139 SNAT_OUT2IN_NEXT_REASS,
141 } snat_out2in_next_t;
144 * @brief Create session for static mapping.
146 * Create NAT session initiated by host from external network with static
149 * @param sm NAT main.
150 * @param b0 Vlib buffer.
151 * @param in2out In2out NAT44 session key.
152 * @param out2in Out2in NAT44 session key.
153 * @param node Vlib node.
155 * @returns SNAT session if successfully created otherwise 0.
157 static inline snat_session_t *
158 create_session_for_static_mapping (snat_main_t *sm,
160 snat_session_key_t in2out,
161 snat_session_key_t out2in,
162 vlib_node_runtime_t * node,
167 clib_bihash_kv_8_8_t kv0;
171 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
173 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
174 nat_log_notice ("maximum sessions exceeded");
178 ip0 = vlib_buffer_get_current (b0);
179 udp0 = ip4_next_header (ip0);
181 u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
184 nat_log_warn ("create NAT user failed");
188 s = nat_session_alloc_or_recycle (sm, u, thread_index);
191 nat_log_warn ("create NAT session failed");
195 s->outside_address_index = ~0;
196 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
197 s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
198 s->ext_host_port = udp0->src_port;
199 user_session_increment (sm, u, 1 /* static */);
202 s->in2out.protocol = out2in.protocol;
204 /* Add to translation hashes */
205 kv0.key = s->in2out.as_u64;
206 kv0.value = s - sm->per_thread_data[thread_index].sessions;
207 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
209 nat_log_notice ("in2out key add failed");
211 kv0.key = s->out2in.as_u64;
213 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
215 nat_log_notice ("out2in key add failed");
218 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
219 s->out2in.addr.as_u32,
223 s->in2out.fib_index);
228 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
229 snat_session_key_t *p_key0)
231 icmp46_header_t *icmp0;
232 snat_session_key_t key0;
233 icmp_echo_header_t *echo0, *inner_echo0 = 0;
234 ip4_header_t *inner_ip0;
236 icmp46_header_t *inner_icmp0;
238 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
239 echo0 = (icmp_echo_header_t *)(icmp0+1);
241 if (!icmp_is_error_message (icmp0))
243 key0.protocol = SNAT_PROTOCOL_ICMP;
244 key0.addr = ip0->dst_address;
245 key0.port = echo0->identifier;
249 inner_ip0 = (ip4_header_t *)(echo0+1);
250 l4_header = ip4_next_header (inner_ip0);
251 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
252 key0.addr = inner_ip0->src_address;
253 switch (key0.protocol)
255 case SNAT_PROTOCOL_ICMP:
256 inner_icmp0 = (icmp46_header_t*)l4_header;
257 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
258 key0.port = inner_echo0->identifier;
260 case SNAT_PROTOCOL_UDP:
261 case SNAT_PROTOCOL_TCP:
262 key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
265 return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
269 return -1; /* success */
273 * Get address and port values to be used for ICMP packet translation
274 * and create session if needed
276 * @param[in,out] sm NAT main
277 * @param[in,out] node NAT node runtime
278 * @param[in] thread_index thread index
279 * @param[in,out] b0 buffer containing packet to be translated
280 * @param[out] p_proto protocol used for matching
281 * @param[out] p_value address and port after NAT translation
282 * @param[out] p_dont_translate if packet should not be translated
283 * @param d optional parameter
284 * @param e optional parameter
286 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
287 u32 thread_index, vlib_buffer_t *b0,
288 ip4_header_t *ip0, u8 *p_proto,
289 snat_session_key_t *p_value,
290 u8 *p_dont_translate, void *d, void *e)
292 icmp46_header_t *icmp0;
295 snat_session_key_t key0;
296 snat_session_key_t sm0;
297 snat_session_t *s0 = 0;
298 u8 dont_translate = 0;
299 clib_bihash_kv_8_8_t kv0, value0;
304 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
305 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
306 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
310 err = icmp_get_key (ip0, &key0);
313 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
314 next0 = SNAT_OUT2IN_NEXT_DROP;
317 key0.fib_index = rx_fib_index0;
319 kv0.key = key0.as_u64;
321 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
324 /* Try to match static mapping by external address and port,
325 destination address and port in packet */
326 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
328 if (!sm->forwarding_enabled)
330 /* Don't NAT packet aimed at the intfc address */
331 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
332 ip0->dst_address.as_u32)))
337 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
338 next0 = SNAT_OUT2IN_NEXT_DROP;
348 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
349 (icmp0->type != ICMP4_echo_request || !is_addr_only)))
351 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
352 next0 = SNAT_OUT2IN_NEXT_DROP;
356 /* Create session initiated by host from external network */
357 s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
362 next0 = SNAT_OUT2IN_NEXT_DROP;
368 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
369 icmp0->type != ICMP4_echo_request &&
370 !icmp_is_error_message (icmp0)))
372 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
373 next0 = SNAT_OUT2IN_NEXT_DROP;
377 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
382 *p_proto = key0.protocol;
384 *p_value = s0->in2out;
385 *p_dont_translate = dont_translate;
387 *(snat_session_t**)d = s0;
392 * Get address and port values to be used for ICMP packet translation
394 * @param[in] sm NAT main
395 * @param[in,out] node NAT node runtime
396 * @param[in] thread_index thread index
397 * @param[in,out] b0 buffer containing packet to be translated
398 * @param[out] p_proto protocol used for matching
399 * @param[out] p_value address and port after NAT translation
400 * @param[out] p_dont_translate if packet should not be translated
401 * @param d optional parameter
402 * @param e optional parameter
404 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
405 u32 thread_index, vlib_buffer_t *b0,
406 ip4_header_t *ip0, u8 *p_proto,
407 snat_session_key_t *p_value,
408 u8 *p_dont_translate, void *d, void *e)
410 icmp46_header_t *icmp0;
413 snat_session_key_t key0;
414 snat_session_key_t sm0;
415 u8 dont_translate = 0;
420 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
421 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
422 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
424 err = icmp_get_key (ip0, &key0);
427 b0->error = node->errors[err];
428 next0 = SNAT_OUT2IN_NEXT_DROP;
431 key0.fib_index = rx_fib_index0;
433 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
435 /* Don't NAT packet aimed at the intfc address */
436 if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
441 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
442 next0 = SNAT_OUT2IN_NEXT_DROP;
446 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
447 (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
448 !icmp_is_error_message (icmp0)))
450 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
451 next0 = SNAT_OUT2IN_NEXT_DROP;
458 *p_proto = key0.protocol;
459 *p_dont_translate = dont_translate;
463 static inline u32 icmp_out2in (snat_main_t *sm,
466 icmp46_header_t * icmp0,
469 vlib_node_runtime_t * node,
475 snat_session_key_t sm0;
477 icmp_echo_header_t *echo0, *inner_echo0 = 0;
478 ip4_header_t *inner_ip0 = 0;
480 icmp46_header_t *inner_icmp0;
482 u32 new_addr0, old_addr0;
483 u16 old_id0, new_id0;
488 echo0 = (icmp_echo_header_t *)(icmp0+1);
490 next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
491 &protocol, &sm0, &dont_translate, d, e);
494 if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
497 sum0 = ip_incremental_checksum (0, icmp0,
498 ntohs(ip0->length) - ip4_header_bytes (ip0));
499 checksum0 = ~ip_csum_fold (sum0);
500 if (checksum0 != 0 && checksum0 != 0xffff)
502 next0 = SNAT_OUT2IN_NEXT_DROP;
506 old_addr0 = ip0->dst_address.as_u32;
507 new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
508 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
510 sum0 = ip0->checksum;
511 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
512 dst_address /* changed member */);
513 ip0->checksum = ip_csum_fold (sum0);
515 if (icmp0->checksum == 0)
516 icmp0->checksum = 0xffff;
518 if (!icmp_is_error_message (icmp0))
521 if (PREDICT_FALSE(new_id0 != echo0->identifier))
523 old_id0 = echo0->identifier;
525 echo0->identifier = new_id0;
527 sum0 = icmp0->checksum;
528 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
529 identifier /* changed member */);
530 icmp0->checksum = ip_csum_fold (sum0);
535 inner_ip0 = (ip4_header_t *)(echo0+1);
536 l4_header = ip4_next_header (inner_ip0);
538 if (!ip4_header_checksum_is_valid (inner_ip0))
540 next0 = SNAT_OUT2IN_NEXT_DROP;
544 old_addr0 = inner_ip0->src_address.as_u32;
545 inner_ip0->src_address = sm0.addr;
546 new_addr0 = inner_ip0->src_address.as_u32;
548 sum0 = icmp0->checksum;
549 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
550 src_address /* changed member */);
551 icmp0->checksum = ip_csum_fold (sum0);
555 case SNAT_PROTOCOL_ICMP:
556 inner_icmp0 = (icmp46_header_t*)l4_header;
557 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
559 old_id0 = inner_echo0->identifier;
561 inner_echo0->identifier = new_id0;
563 sum0 = icmp0->checksum;
564 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
566 icmp0->checksum = ip_csum_fold (sum0);
568 case SNAT_PROTOCOL_UDP:
569 case SNAT_PROTOCOL_TCP:
570 old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
572 ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
574 sum0 = icmp0->checksum;
575 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
577 icmp0->checksum = ip_csum_fold (sum0);
589 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
592 icmp46_header_t * icmp0,
595 vlib_node_runtime_t * node,
598 snat_session_t ** p_s0)
600 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
601 next0, thread_index, p_s0, 0);
602 snat_session_t * s0 = *p_s0;
603 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
606 nat44_session_update_counters (s0, now,
607 vlib_buffer_length_in_chain (sm->vlib_main, b0));
608 /* Per-user LRU list maintenance */
609 nat44_session_update_lru (sm, s0, thread_index);
615 nat_out2in_sm_unknown_proto (snat_main_t *sm,
620 clib_bihash_kv_8_8_t kv, value;
621 snat_static_mapping_t *m;
622 snat_session_key_t m_key;
623 u32 old_addr, new_addr;
626 m_key.addr = ip->dst_address;
630 kv.key = m_key.as_u64;
631 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
634 m = pool_elt_at_index (sm->static_mappings, value.value);
636 old_addr = ip->dst_address.as_u32;
637 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
639 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
640 ip->checksum = ip_csum_fold (sum);
642 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
647 snat_out2in_node_fn (vlib_main_t * vm,
648 vlib_node_runtime_t * node,
649 vlib_frame_t * frame)
651 u32 n_left_from, * from, * to_next;
652 snat_out2in_next_t next_index;
653 u32 pkts_processed = 0;
654 snat_main_t * sm = &snat_main;
655 f64 now = vlib_time_now (vm);
656 u32 thread_index = vm->thread_index;
658 from = vlib_frame_vector_args (frame);
659 n_left_from = frame->n_vectors;
660 next_index = node->cached_next_index;
662 while (n_left_from > 0)
666 vlib_get_next_frame (vm, node, next_index,
667 to_next, n_left_to_next);
669 while (n_left_from >= 4 && n_left_to_next >= 2)
672 vlib_buffer_t * b0, * b1;
673 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
674 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
675 u32 sw_if_index0, sw_if_index1;
676 ip4_header_t * ip0, *ip1;
677 ip_csum_t sum0, sum1;
678 u32 new_addr0, old_addr0;
679 u16 new_port0, old_port0;
680 u32 new_addr1, old_addr1;
681 u16 new_port1, old_port1;
682 udp_header_t * udp0, * udp1;
683 tcp_header_t * tcp0, * tcp1;
684 icmp46_header_t * icmp0, * icmp1;
685 snat_session_key_t key0, key1, sm0, sm1;
686 u32 rx_fib_index0, rx_fib_index1;
688 snat_session_t * s0 = 0, * s1 = 0;
689 clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
691 /* Prefetch next iteration. */
693 vlib_buffer_t * p2, * p3;
695 p2 = vlib_get_buffer (vm, from[2]);
696 p3 = vlib_get_buffer (vm, from[3]);
698 vlib_prefetch_buffer_header (p2, LOAD);
699 vlib_prefetch_buffer_header (p3, LOAD);
701 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
702 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
705 /* speculatively enqueue b0 and b1 to the current next frame */
706 to_next[0] = bi0 = from[0];
707 to_next[1] = bi1 = from[1];
713 b0 = vlib_get_buffer (vm, bi0);
714 b1 = vlib_get_buffer (vm, bi1);
716 vnet_buffer (b0)->snat.flags = 0;
717 vnet_buffer (b1)->snat.flags = 0;
719 ip0 = vlib_buffer_get_current (b0);
720 udp0 = ip4_next_header (ip0);
721 tcp0 = (tcp_header_t *) udp0;
722 icmp0 = (icmp46_header_t *) udp0;
724 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
725 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
728 if (PREDICT_FALSE(ip0->ttl == 1))
730 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
731 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
732 ICMP4_time_exceeded_ttl_exceeded_in_transit,
734 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
738 proto0 = ip_proto_to_snat_proto (ip0->protocol);
740 if (PREDICT_FALSE (proto0 == ~0))
742 if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
744 if (!sm->forwarding_enabled)
746 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
747 next0 = SNAT_OUT2IN_NEXT_DROP;
753 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
755 next0 = icmp_out2in_slow_path
756 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
757 next0, now, thread_index, &s0);
761 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
763 next0 = SNAT_OUT2IN_NEXT_REASS;
767 key0.addr = ip0->dst_address;
768 key0.port = udp0->dst_port;
769 key0.protocol = proto0;
770 key0.fib_index = rx_fib_index0;
772 kv0.key = key0.as_u64;
774 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
777 /* Try to match static mapping by external address and port,
778 destination address and port in packet */
779 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
782 * Send DHCP packets to the ipv4 stack, or we won't
783 * be able to use dhcp client on the outside interface
785 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
786 && (udp0->dst_port ==
787 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
790 (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
794 if (!sm->forwarding_enabled)
796 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
797 next0 = SNAT_OUT2IN_NEXT_DROP;
802 /* Create session initiated by host from external network */
803 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
807 next0 = SNAT_OUT2IN_NEXT_DROP;
812 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
815 old_addr0 = ip0->dst_address.as_u32;
816 ip0->dst_address = s0->in2out.addr;
817 new_addr0 = ip0->dst_address.as_u32;
818 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
820 sum0 = ip0->checksum;
821 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
823 dst_address /* changed member */);
824 ip0->checksum = ip_csum_fold (sum0);
826 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
828 old_port0 = tcp0->dst_port;
829 tcp0->dst_port = s0->in2out.port;
830 new_port0 = tcp0->dst_port;
832 sum0 = tcp0->checksum;
833 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
835 dst_address /* changed member */);
837 sum0 = ip_csum_update (sum0, old_port0, new_port0,
838 ip4_header_t /* cheat */,
839 length /* changed member */);
840 tcp0->checksum = ip_csum_fold(sum0);
844 old_port0 = udp0->dst_port;
845 udp0->dst_port = s0->in2out.port;
850 nat44_session_update_counters (s0, now,
851 vlib_buffer_length_in_chain (vm, b0));
852 /* Per-user LRU list maintenance */
853 nat44_session_update_lru (sm, s0, thread_index);
856 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
857 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
859 snat_out2in_trace_t *t =
860 vlib_add_trace (vm, node, b0, sizeof (*t));
861 t->sw_if_index = sw_if_index0;
862 t->next_index = next0;
863 t->session_index = ~0;
865 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
868 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
871 ip1 = vlib_buffer_get_current (b1);
872 udp1 = ip4_next_header (ip1);
873 tcp1 = (tcp_header_t *) udp1;
874 icmp1 = (icmp46_header_t *) udp1;
876 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
877 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
880 if (PREDICT_FALSE(ip1->ttl == 1))
882 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
883 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
884 ICMP4_time_exceeded_ttl_exceeded_in_transit,
886 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
890 proto1 = ip_proto_to_snat_proto (ip1->protocol);
892 if (PREDICT_FALSE (proto1 == ~0))
894 if (nat_out2in_sm_unknown_proto(sm, b1, ip1, rx_fib_index1))
896 if (!sm->forwarding_enabled)
898 b1->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
899 next1 = SNAT_OUT2IN_NEXT_DROP;
905 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
907 next1 = icmp_out2in_slow_path
908 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
909 next1, now, thread_index, &s1);
913 if (PREDICT_FALSE (ip4_is_fragment (ip1)))
915 next1 = SNAT_OUT2IN_NEXT_REASS;
919 key1.addr = ip1->dst_address;
920 key1.port = udp1->dst_port;
921 key1.protocol = proto1;
922 key1.fib_index = rx_fib_index1;
924 kv1.key = key1.as_u64;
926 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
929 /* Try to match static mapping by external address and port,
930 destination address and port in packet */
931 if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
934 * Send DHCP packets to the ipv4 stack, or we won't
935 * be able to use dhcp client on the outside interface
937 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
938 && (udp1->dst_port ==
939 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
942 (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
946 if (!sm->forwarding_enabled)
948 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
949 next1 = SNAT_OUT2IN_NEXT_DROP;
954 /* Create session initiated by host from external network */
955 s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
959 next1 = SNAT_OUT2IN_NEXT_DROP;
964 s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
967 old_addr1 = ip1->dst_address.as_u32;
968 ip1->dst_address = s1->in2out.addr;
969 new_addr1 = ip1->dst_address.as_u32;
970 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
972 sum1 = ip1->checksum;
973 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
975 dst_address /* changed member */);
976 ip1->checksum = ip_csum_fold (sum1);
978 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
980 old_port1 = tcp1->dst_port;
981 tcp1->dst_port = s1->in2out.port;
982 new_port1 = tcp1->dst_port;
984 sum1 = tcp1->checksum;
985 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
987 dst_address /* changed member */);
989 sum1 = ip_csum_update (sum1, old_port1, new_port1,
990 ip4_header_t /* cheat */,
991 length /* changed member */);
992 tcp1->checksum = ip_csum_fold(sum1);
996 old_port1 = udp1->dst_port;
997 udp1->dst_port = s1->in2out.port;
1002 nat44_session_update_counters (s1, now,
1003 vlib_buffer_length_in_chain (vm, b1));
1004 /* Per-user LRU list maintenance */
1005 nat44_session_update_lru (sm, s1, thread_index);
1008 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1009 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1011 snat_out2in_trace_t *t =
1012 vlib_add_trace (vm, node, b1, sizeof (*t));
1013 t->sw_if_index = sw_if_index1;
1014 t->next_index = next1;
1015 t->session_index = ~0;
1017 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1020 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1022 /* verify speculative enqueues, maybe switch current next frame */
1023 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1024 to_next, n_left_to_next,
1025 bi0, bi1, next0, next1);
1028 while (n_left_from > 0 && n_left_to_next > 0)
1032 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1036 u32 new_addr0, old_addr0;
1037 u16 new_port0, old_port0;
1038 udp_header_t * udp0;
1039 tcp_header_t * tcp0;
1040 icmp46_header_t * icmp0;
1041 snat_session_key_t key0, sm0;
1044 snat_session_t * s0 = 0;
1045 clib_bihash_kv_8_8_t kv0, value0;
1047 /* speculatively enqueue b0 to the current next frame */
1053 n_left_to_next -= 1;
1055 b0 = vlib_get_buffer (vm, bi0);
1057 vnet_buffer (b0)->snat.flags = 0;
1059 ip0 = vlib_buffer_get_current (b0);
1060 udp0 = ip4_next_header (ip0);
1061 tcp0 = (tcp_header_t *) udp0;
1062 icmp0 = (icmp46_header_t *) udp0;
1064 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1065 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1068 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1070 if (PREDICT_FALSE (proto0 == ~0))
1072 if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
1074 if (!sm->forwarding_enabled)
1076 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1077 next0 = SNAT_OUT2IN_NEXT_DROP;
1083 if (PREDICT_FALSE(ip0->ttl == 1))
1085 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1086 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1087 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1089 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1093 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1095 next0 = icmp_out2in_slow_path
1096 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1097 next0, now, thread_index, &s0);
1101 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1103 next0 = SNAT_OUT2IN_NEXT_REASS;
1107 key0.addr = ip0->dst_address;
1108 key0.port = udp0->dst_port;
1109 key0.protocol = proto0;
1110 key0.fib_index = rx_fib_index0;
1112 kv0.key = key0.as_u64;
1114 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1117 /* Try to match static mapping by external address and port,
1118 destination address and port in packet */
1119 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1122 * Send DHCP packets to the ipv4 stack, or we won't
1123 * be able to use dhcp client on the outside interface
1125 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1126 && (udp0->dst_port ==
1127 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1130 (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
1134 if (!sm->forwarding_enabled)
1136 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1137 next0 = SNAT_OUT2IN_NEXT_DROP;
1142 /* Create session initiated by host from external network */
1143 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1147 next0 = SNAT_OUT2IN_NEXT_DROP;
1152 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1155 old_addr0 = ip0->dst_address.as_u32;
1156 ip0->dst_address = s0->in2out.addr;
1157 new_addr0 = ip0->dst_address.as_u32;
1158 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1160 sum0 = ip0->checksum;
1161 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1163 dst_address /* changed member */);
1164 ip0->checksum = ip_csum_fold (sum0);
1166 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1168 old_port0 = tcp0->dst_port;
1169 tcp0->dst_port = s0->in2out.port;
1170 new_port0 = tcp0->dst_port;
1172 sum0 = tcp0->checksum;
1173 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1175 dst_address /* changed member */);
1177 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1178 ip4_header_t /* cheat */,
1179 length /* changed member */);
1180 tcp0->checksum = ip_csum_fold(sum0);
1184 old_port0 = udp0->dst_port;
1185 udp0->dst_port = s0->in2out.port;
1190 nat44_session_update_counters (s0, now,
1191 vlib_buffer_length_in_chain (vm, b0));
1192 /* Per-user LRU list maintenance */
1193 nat44_session_update_lru (sm, s0, thread_index);
1196 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1197 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1199 snat_out2in_trace_t *t =
1200 vlib_add_trace (vm, node, b0, sizeof (*t));
1201 t->sw_if_index = sw_if_index0;
1202 t->next_index = next0;
1203 t->session_index = ~0;
1205 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1208 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1210 /* verify speculative enqueue, maybe switch current next frame */
1211 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1212 to_next, n_left_to_next,
1216 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1219 vlib_node_increment_counter (vm, snat_out2in_node.index,
1220 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1222 return frame->n_vectors;
1225 VLIB_REGISTER_NODE (snat_out2in_node) = {
1226 .function = snat_out2in_node_fn,
1227 .name = "nat44-out2in",
1228 .vector_size = sizeof (u32),
1229 .format_trace = format_snat_out2in_trace,
1230 .type = VLIB_NODE_TYPE_INTERNAL,
1232 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1233 .error_strings = snat_out2in_error_strings,
1235 .runtime_data_bytes = sizeof (snat_runtime_t),
1237 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1239 /* edit / add dispositions here */
1241 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1242 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1243 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1244 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1247 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1250 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1251 vlib_node_runtime_t * node,
1252 vlib_frame_t * frame)
1254 u32 n_left_from, *from, *to_next;
1255 snat_out2in_next_t next_index;
1256 u32 pkts_processed = 0;
1257 snat_main_t *sm = &snat_main;
1258 f64 now = vlib_time_now (vm);
1259 u32 thread_index = vm->thread_index;
1260 snat_main_per_thread_data_t *per_thread_data =
1261 &sm->per_thread_data[thread_index];
1262 u32 *fragments_to_drop = 0;
1263 u32 *fragments_to_loopback = 0;
1265 from = vlib_frame_vector_args (frame);
1266 n_left_from = frame->n_vectors;
1267 next_index = node->cached_next_index;
1269 while (n_left_from > 0)
1273 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1275 while (n_left_from > 0 && n_left_to_next > 0)
1277 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1282 nat_reass_ip4_t *reass0;
1283 udp_header_t * udp0;
1284 tcp_header_t * tcp0;
1285 snat_session_key_t key0, sm0;
1286 clib_bihash_kv_8_8_t kv0, value0;
1287 snat_session_t * s0 = 0;
1288 u16 old_port0, new_port0;
1291 /* speculatively enqueue b0 to the current next frame */
1297 n_left_to_next -= 1;
1299 b0 = vlib_get_buffer (vm, bi0);
1300 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1302 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1303 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1306 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1308 next0 = SNAT_OUT2IN_NEXT_DROP;
1309 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1313 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1314 udp0 = ip4_next_header (ip0);
1315 tcp0 = (tcp_header_t *) udp0;
1316 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1318 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1323 &fragments_to_drop);
1325 if (PREDICT_FALSE (!reass0))
1327 next0 = SNAT_OUT2IN_NEXT_DROP;
1328 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1329 nat_log_notice ("maximum reassemblies exceeded");
1333 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1335 key0.addr = ip0->dst_address;
1336 key0.port = udp0->dst_port;
1337 key0.protocol = proto0;
1338 key0.fib_index = rx_fib_index0;
1339 kv0.key = key0.as_u64;
1341 if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1343 /* Try to match static mapping by external address and port,
1344 destination address and port in packet */
1345 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1348 * Send DHCP packets to the ipv4 stack, or we won't
1349 * be able to use dhcp client on the outside interface
1351 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1353 == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1356 (vnet_buffer (b0)->sw_if_index[VLIB_RX],
1361 if (!sm->forwarding_enabled)
1363 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1364 next0 = SNAT_OUT2IN_NEXT_DROP;
1369 /* Create session initiated by host from external network */
1370 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1374 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1375 next0 = SNAT_OUT2IN_NEXT_DROP;
1378 reass0->sess_index = s0 - per_thread_data->sessions;
1379 reass0->thread_index = thread_index;
1383 s0 = pool_elt_at_index (per_thread_data->sessions,
1385 reass0->sess_index = value0.value;
1387 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1391 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1393 if (nat_ip4_reass_add_fragment (reass0, bi0))
1395 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1396 nat_log_notice ("maximum fragments per reassembly exceeded");
1397 next0 = SNAT_OUT2IN_NEXT_DROP;
1403 s0 = pool_elt_at_index (per_thread_data->sessions,
1404 reass0->sess_index);
1407 old_addr0 = ip0->dst_address.as_u32;
1408 ip0->dst_address = s0->in2out.addr;
1409 new_addr0 = ip0->dst_address.as_u32;
1410 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1412 sum0 = ip0->checksum;
1413 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1415 dst_address /* changed member */);
1416 ip0->checksum = ip_csum_fold (sum0);
1418 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1420 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1422 old_port0 = tcp0->dst_port;
1423 tcp0->dst_port = s0->in2out.port;
1424 new_port0 = tcp0->dst_port;
1426 sum0 = tcp0->checksum;
1427 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1429 dst_address /* changed member */);
1431 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1432 ip4_header_t /* cheat */,
1433 length /* changed member */);
1434 tcp0->checksum = ip_csum_fold(sum0);
1438 old_port0 = udp0->dst_port;
1439 udp0->dst_port = s0->in2out.port;
1445 nat44_session_update_counters (s0, now,
1446 vlib_buffer_length_in_chain (vm, b0));
1447 /* Per-user LRU list maintenance */
1448 nat44_session_update_lru (sm, s0, thread_index);
1451 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1452 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1454 nat44_out2in_reass_trace_t *t =
1455 vlib_add_trace (vm, node, b0, sizeof (*t));
1456 t->cached = cached0;
1457 t->sw_if_index = sw_if_index0;
1458 t->next_index = next0;
1468 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1470 /* verify speculative enqueue, maybe switch current next frame */
1471 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1472 to_next, n_left_to_next,
1476 if (n_left_from == 0 && vec_len (fragments_to_loopback))
1478 from = vlib_frame_vector_args (frame);
1479 u32 len = vec_len (fragments_to_loopback);
1480 if (len <= VLIB_FRAME_SIZE)
1482 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1484 vec_reset_length (fragments_to_loopback);
1489 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1490 sizeof (u32) * VLIB_FRAME_SIZE);
1491 n_left_from = VLIB_FRAME_SIZE;
1492 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1497 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1500 vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1501 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1504 nat_send_all_to_node (vm, fragments_to_drop, node,
1505 &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1506 SNAT_OUT2IN_NEXT_DROP);
1508 vec_free (fragments_to_drop);
1509 vec_free (fragments_to_loopback);
1510 return frame->n_vectors;
1513 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1514 .function = nat44_out2in_reass_node_fn,
1515 .name = "nat44-out2in-reass",
1516 .vector_size = sizeof (u32),
1517 .format_trace = format_nat44_out2in_reass_trace,
1518 .type = VLIB_NODE_TYPE_INTERNAL,
1520 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1521 .error_strings = snat_out2in_error_strings,
1523 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1525 /* edit / add dispositions here */
1527 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1528 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1529 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1530 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1533 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1534 nat44_out2in_reass_node_fn);
1536 /*******************************/
1537 /*** endpoint-dependent mode ***/
1538 /*******************************/
1540 NAT44_ED_OUT2IN_NEXT_DROP,
1541 NAT44_ED_OUT2IN_NEXT_LOOKUP,
1542 NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
1543 NAT44_ED_OUT2IN_NEXT_IN2OUT,
1544 NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
1545 NAT44_ED_OUT2IN_N_NEXT,
1546 } nat44_ed_out2in_next_t;
1553 } nat44_ed_out2in_trace_t;
1556 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
1558 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1559 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1560 nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
1563 tag = t->is_slow_path ? "NAT44_OUT2IN_SLOW_PATH" : "NAT44_OUT2IN_FAST_PATH";
1565 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
1566 t->sw_if_index, t->next_index, t->session_index);
1571 static snat_session_t *
1572 create_session_for_static_mapping_ed (snat_main_t * sm,
1574 snat_session_key_t l_key,
1575 snat_session_key_t e_key,
1576 vlib_node_runtime_t * node,
1578 twice_nat_type_t twice_nat,
1585 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1586 clib_bihash_kv_16_8_t kv;
1587 snat_session_key_t eh_key;
1590 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1592 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1593 nat_log_notice ("maximum sessions exceeded");
1597 u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
1600 nat_log_warn ("create NAT user failed");
1604 s = nat_session_alloc_or_recycle (sm, u, thread_index);
1607 nat_log_warn ("create NAT session failed");
1611 ip = vlib_buffer_get_current (b);
1612 udp = ip4_next_header (ip);
1614 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1615 s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
1616 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1618 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1619 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1620 s->outside_address_index = ~0;
1623 s->in2out.protocol = s->out2in.protocol;
1624 user_session_increment (sm, u, 1);
1626 /* Add to lookup tables */
1627 make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
1628 e_key.fib_index, e_key.port, s->ext_host_port);
1629 kv.value = s - tsm->sessions;
1630 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
1631 nat_log_notice ("out2in-ed key add failed");
1633 if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
1634 ip->src_address.as_u32 == l_key.addr.as_u32))
1636 eh_key.protocol = e_key.protocol;
1637 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1638 thread_index, &eh_key,
1640 sm->port_per_thread,
1641 tsm->snat_thread_index))
1643 b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1644 nat44_delete_session (sm, s, thread_index);
1645 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
1646 nat_log_notice ("out2in-ed key del failed");
1649 s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1650 s->ext_host_nat_port = eh_key.port;
1651 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1652 make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
1653 l_key.fib_index, l_key.port, s->ext_host_nat_port);
1657 make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
1658 l_key.fib_index, l_key.port, s->ext_host_port);
1660 kv.value = s - tsm->sessions;
1661 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1662 nat_log_notice ("in2out-ed key add failed");
1667 static_always_inline int
1668 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
1670 icmp46_header_t *icmp0;
1671 nat_ed_ses_key_t key0;
1672 icmp_echo_header_t *echo0, *inner_echo0 = 0;
1673 ip4_header_t *inner_ip0;
1674 void *l4_header = 0;
1675 icmp46_header_t *inner_icmp0;
1677 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1678 echo0 = (icmp_echo_header_t *)(icmp0+1);
1680 if (!icmp_is_error_message (icmp0))
1682 key0.proto = IP_PROTOCOL_ICMP;
1683 key0.l_addr = ip0->dst_address;
1684 key0.r_addr = ip0->src_address;
1685 key0.l_port = echo0->identifier;
1690 inner_ip0 = (ip4_header_t *)(echo0+1);
1691 l4_header = ip4_next_header (inner_ip0);
1692 key0.proto = inner_ip0->protocol;
1693 key0.l_addr = inner_ip0->src_address;
1694 key0.r_addr = inner_ip0->dst_address;
1695 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
1697 case SNAT_PROTOCOL_ICMP:
1698 inner_icmp0 = (icmp46_header_t*)l4_header;
1699 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1700 key0.l_port = inner_echo0->identifier;
1703 case SNAT_PROTOCOL_UDP:
1704 case SNAT_PROTOCOL_TCP:
1705 key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
1706 key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1717 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
1718 u16 dst_port, u32 thread_index)
1720 clib_bihash_kv_16_8_t kv, value;
1721 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1723 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
1724 sm->inside_fib_index, src_port, dst_port);
1725 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1732 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
1735 nat_ed_ses_key_t key;
1736 clib_bihash_kv_16_8_t kv, value;
1739 snat_session_t *s = 0;
1740 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1741 f64 now = vlib_time_now (sm->vlib_main);
1743 if (ip->protocol == IP_PROTOCOL_ICMP)
1745 if (icmp_get_ed_key (ip, &key))
1748 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
1750 udp = ip4_next_header(ip);
1751 key.r_addr = ip->src_address;
1752 key.l_addr = ip->dst_address;
1753 key.proto = ip->protocol;
1754 key.l_port = udp->dst_port;
1755 key.r_port = udp->src_port;
1759 key.r_addr = ip->src_address;
1760 key.l_addr = ip->dst_address;
1761 key.proto = ip->protocol;
1762 key.l_port = key.r_port = 0;
1765 kv.key[0] = key.as_u64[0];
1766 kv.key[1] = key.as_u64[1];
1768 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1770 s = pool_elt_at_index (tsm->sessions, value.value);
1774 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1777 u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
1781 nat_log_warn ("create NAT user failed");
1785 s = nat_session_alloc_or_recycle (sm, u, thread_index);
1788 nat_log_warn ("create NAT session failed");
1792 s->ext_host_addr = key.r_addr;
1793 s->ext_host_port = key.r_port;
1794 s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
1795 s->outside_address_index = ~0;
1796 s->out2in.addr = key.l_addr;
1797 s->out2in.port = key.l_port;
1798 s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
1799 s->out2in.fib_index = 0;
1800 s->in2out = s->out2in;
1801 user_session_increment (sm, u, 0);
1803 kv.value = s - tsm->sessions;
1804 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1805 nat_log_notice ("in2out_ed key add failed");
1808 if (ip->protocol == IP_PROTOCOL_TCP)
1810 tcp_header_t *tcp = ip4_next_header(ip);
1811 if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
1815 /* Per-user LRU list maintenance */
1816 nat44_session_update_lru (sm, s, thread_index);
1818 nat44_session_update_counters (s, now, 0);
1822 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
1823 u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
1824 u8 * p_proto, snat_session_key_t * p_value,
1825 u8 * p_dont_translate, void * d, void * e)
1827 u32 next = ~0, sw_if_index, rx_fib_index;
1828 icmp46_header_t *icmp;
1829 nat_ed_ses_key_t key;
1830 clib_bihash_kv_16_8_t kv, value;
1831 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1832 snat_session_t *s = 0;
1833 u8 dont_translate = 0, is_addr_only;
1834 snat_session_key_t e_key, l_key;
1836 icmp = (icmp46_header_t *) ip4_next_header (ip);
1837 sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
1838 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
1840 if (icmp_get_ed_key (ip, &key))
1842 b->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1843 next = SNAT_OUT2IN_NEXT_DROP;
1846 key.fib_index = rx_fib_index;
1847 kv.key[0] = key.as_u64[0];
1848 kv.key[1] = key.as_u64[1];
1850 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
1852 /* Try to match static mapping */
1853 e_key.addr = ip->dst_address;
1854 e_key.port = key.l_port;
1855 e_key.protocol = ip_proto_to_snat_proto (key.proto);
1856 e_key.fib_index = rx_fib_index;
1857 if (snat_static_mapping_match(sm, e_key, &l_key, 1, &is_addr_only, 0, 0))
1859 if (!sm->forwarding_enabled)
1861 /* Don't NAT packet aimed at the intfc address */
1862 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index,
1863 ip->dst_address.as_u32)))
1868 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1869 next = NAT44_ED_OUT2IN_NEXT_DROP;
1875 if (next_src_nat(sm, ip, key.proto, key.l_port, key.r_port, thread_index))
1877 next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1880 create_bypass_for_fwd(sm, ip, rx_fib_index, thread_index);
1885 if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1886 (icmp->type != ICMP4_echo_request || !is_addr_only)))
1888 b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1889 next = NAT44_ED_OUT2IN_NEXT_DROP;
1893 /* Create session initiated by host from external network */
1894 s = create_session_for_static_mapping_ed(sm, b, l_key, e_key, node,
1895 thread_index, 0, 0);
1899 next = NAT44_ED_OUT2IN_NEXT_DROP;
1905 if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1906 icmp->type != ICMP4_echo_request &&
1907 !icmp_is_error_message (icmp)))
1909 b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1910 next = SNAT_OUT2IN_NEXT_DROP;
1914 s = pool_elt_at_index (tsm->sessions, value.value);
1917 *p_proto = ip_proto_to_snat_proto (key.proto);
1920 *p_value = s->in2out;
1921 *p_dont_translate = dont_translate;
1923 *(snat_session_t**)d = s;
1927 static snat_session_t *
1928 nat44_ed_out2in_unknown_proto (snat_main_t *sm,
1935 vlib_node_runtime_t * node)
1937 clib_bihash_kv_8_8_t kv, value;
1938 clib_bihash_kv_16_8_t s_kv, s_value;
1939 snat_static_mapping_t *m;
1940 u32 old_addr, new_addr;
1943 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1946 old_addr = ip->dst_address.as_u32;
1948 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
1949 rx_fib_index, 0, 0);
1951 if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
1953 s = pool_elt_at_index (tsm->sessions, s_value.value);
1954 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1958 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1960 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1961 nat_log_notice ("maximum sessions exceeded");
1965 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1966 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1968 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1972 m = pool_elt_at_index (sm->static_mappings, value.value);
1974 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1976 u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
1980 nat_log_warn ("create NAT user failed");
1984 /* Create a new session */
1985 s = nat_session_alloc_or_recycle (sm, u, thread_index);
1988 nat_log_warn ("create NAT session failed");
1992 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1993 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1994 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1995 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1996 s->outside_address_index = ~0;
1997 s->out2in.addr.as_u32 = old_addr;
1998 s->out2in.fib_index = rx_fib_index;
1999 s->in2out.addr.as_u32 = new_addr;
2000 s->in2out.fib_index = m->fib_index;
2001 s->in2out.port = s->out2in.port = ip->protocol;
2002 user_session_increment (sm, u, 1);
2004 /* Add to lookup tables */
2005 s_kv.value = s - tsm->sessions;
2006 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2007 nat_log_notice ("out2in key add failed");
2009 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2010 m->fib_index, 0, 0);
2011 s_kv.value = s - tsm->sessions;
2012 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2013 nat_log_notice ("in2out key add failed");
2016 /* Update IP checksum */
2018 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2019 ip->checksum = ip_csum_fold (sum);
2021 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2024 nat44_session_update_counters (s, now,
2025 vlib_buffer_length_in_chain (vm, b));
2026 /* Per-user LRU list maintenance */
2027 nat44_session_update_lru (sm, s, thread_index);
2033 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
2034 vlib_node_runtime_t * node,
2035 vlib_frame_t * frame, int is_slow_path)
2037 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2038 nat44_ed_out2in_next_t next_index;
2039 snat_main_t *sm = &snat_main;
2040 f64 now = vlib_time_now (vm);
2041 u32 thread_index = vm->thread_index;
2042 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2044 stats_node_index = is_slow_path ? nat44_ed_out2in_slowpath_node.index :
2045 nat44_ed_out2in_node.index;
2047 from = vlib_frame_vector_args (frame);
2048 n_left_from = frame->n_vectors;
2049 next_index = node->cached_next_index;
2051 while (n_left_from > 0)
2055 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2057 while (n_left_from >= 4 && n_left_to_next >= 2)
2060 vlib_buffer_t *b0, *b1;
2061 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2062 u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1, new_addr1;
2063 u16 old_port0, new_port0, old_port1, new_port1;
2064 ip4_header_t *ip0, *ip1;
2065 udp_header_t *udp0, *udp1;
2066 tcp_header_t *tcp0, *tcp1;
2067 icmp46_header_t *icmp0, *icmp1;
2068 snat_session_t *s0 = 0, *s1 = 0;
2069 clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
2070 ip_csum_t sum0, sum1;
2071 snat_session_key_t e_key0, l_key0, e_key1, l_key1;
2073 twice_nat_type_t twice_nat0, twice_nat1;
2075 /* Prefetch next iteration. */
2077 vlib_buffer_t * p2, * p3;
2079 p2 = vlib_get_buffer (vm, from[2]);
2080 p3 = vlib_get_buffer (vm, from[3]);
2082 vlib_prefetch_buffer_header (p2, LOAD);
2083 vlib_prefetch_buffer_header (p3, LOAD);
2085 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2086 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2089 /* speculatively enqueue b0 and b1 to the current next frame */
2090 to_next[0] = bi0 = from[0];
2091 to_next[1] = bi1 = from[1];
2095 n_left_to_next -= 2;
2097 b0 = vlib_get_buffer (vm, bi0);
2098 b1 = vlib_get_buffer (vm, bi1);
2100 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2101 vnet_buffer (b0)->snat.flags = 0;
2102 ip0 = vlib_buffer_get_current (b0);
2104 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2105 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2108 if (PREDICT_FALSE(ip0->ttl == 1))
2110 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2111 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2112 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2114 next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2118 udp0 = ip4_next_header (ip0);
2119 tcp0 = (tcp_header_t *) udp0;
2120 icmp0 = (icmp46_header_t *) udp0;
2121 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2125 if (PREDICT_FALSE (proto0 == ~0))
2127 s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2128 thread_index, now, vm, node);
2129 if (!sm->forwarding_enabled)
2132 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2137 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2139 next0 = icmp_out2in_slow_path
2140 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2141 next0, now, thread_index, &s0);
2147 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2149 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2153 if (ip4_is_fragment (ip0))
2155 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2156 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2161 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2162 rx_fib_index0, udp0->dst_port, udp0->src_port);
2164 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2168 /* Try to match static mapping by external address and port,
2169 destination address and port in packet */
2170 e_key0.addr = ip0->dst_address;
2171 e_key0.port = udp0->dst_port;
2172 e_key0.protocol = proto0;
2173 e_key0.fib_index = rx_fib_index0;
2174 if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2175 &twice_nat0, &is_lb0))
2178 * Send DHCP packets to the ipv4 stack, or we won't
2179 * be able to use dhcp client on the outside interface
2181 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2182 && (udp0->dst_port ==
2183 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2186 (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
2190 if (!sm->forwarding_enabled)
2192 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2193 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2197 if (next_src_nat(sm, ip0, ip0->protocol,
2198 udp0->src_port, udp0->dst_port,
2201 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2204 create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2210 /* Create session initiated by host from external network */
2211 s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2214 twice_nat0, is_lb0);
2218 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2224 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2230 s0 = pool_elt_at_index (tsm->sessions, value0.value);
2233 old_addr0 = ip0->dst_address.as_u32;
2234 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2235 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2237 sum0 = ip0->checksum;
2238 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2240 if (PREDICT_FALSE (is_twice_nat_session (s0)))
2241 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2242 s0->ext_host_nat_addr.as_u32, ip4_header_t,
2244 ip0->checksum = ip_csum_fold (sum0);
2246 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2248 old_port0 = tcp0->dst_port;
2249 new_port0 = tcp0->dst_port = s0->in2out.port;
2251 sum0 = tcp0->checksum;
2252 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2254 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2256 if (is_twice_nat_session (s0))
2258 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2259 s0->ext_host_nat_addr.as_u32,
2260 ip4_header_t, dst_address);
2261 sum0 = ip_csum_update (sum0, tcp0->src_port,
2262 s0->ext_host_nat_port, ip4_header_t,
2264 tcp0->src_port = s0->ext_host_nat_port;
2265 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2267 tcp0->checksum = ip_csum_fold(sum0);
2268 if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2273 udp0->dst_port = s0->in2out.port;
2274 if (is_twice_nat_session (s0))
2276 udp0->src_port = s0->ext_host_nat_port;
2277 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2283 nat44_session_update_counters (s0, now,
2284 vlib_buffer_length_in_chain (vm, b0));
2285 /* Per-user LRU list maintenance */
2286 nat44_session_update_lru (sm, s0, thread_index);
2289 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2290 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2292 nat44_ed_out2in_trace_t *t =
2293 vlib_add_trace (vm, node, b0, sizeof (*t));
2294 t->is_slow_path = is_slow_path;
2295 t->sw_if_index = sw_if_index0;
2296 t->next_index = next0;
2297 t->session_index = ~0;
2299 t->session_index = s0 - tsm->sessions;
2302 pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2304 next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2305 vnet_buffer (b1)->snat.flags = 0;
2306 ip1 = vlib_buffer_get_current (b1);
2308 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2309 rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2312 if (PREDICT_FALSE(ip1->ttl == 1))
2314 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2315 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2316 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2318 next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2322 udp1 = ip4_next_header (ip1);
2323 tcp1 = (tcp_header_t *) udp1;
2324 icmp1 = (icmp46_header_t *) udp1;
2325 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2329 if (PREDICT_FALSE (proto1 == ~0))
2331 s1 = nat44_ed_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
2332 thread_index, now, vm, node);
2333 if (!sm->forwarding_enabled)
2336 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2341 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
2343 next1 = icmp_out2in_slow_path
2344 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
2345 next1, now, thread_index, &s1);
2351 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
2353 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2357 if (ip4_is_fragment (ip1))
2359 b1->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2360 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2365 make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address, ip1->protocol,
2366 rx_fib_index1, udp1->dst_port, udp1->src_port);
2368 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
2372 /* Try to match static mapping by external address and port,
2373 destination address and port in packet */
2374 e_key1.addr = ip1->dst_address;
2375 e_key1.port = udp1->dst_port;
2376 e_key1.protocol = proto1;
2377 e_key1.fib_index = rx_fib_index1;
2378 if (snat_static_mapping_match(sm, e_key1, &l_key1, 1, 0,
2379 &twice_nat1, &is_lb1))
2382 * Send DHCP packets to the ipv4 stack, or we won't
2383 * be able to use dhcp client on the outside interface
2385 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
2386 && (udp1->dst_port ==
2387 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2390 (vnet_buffer (b1)->sw_if_index[VLIB_RX], &next1, b1);
2394 if (!sm->forwarding_enabled)
2396 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2397 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2401 if (next_src_nat(sm, ip1, ip1->protocol,
2402 udp1->src_port, udp1->dst_port,
2405 next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2408 create_bypass_for_fwd(sm, ip1, rx_fib_index1,
2414 /* Create session initiated by host from external network */
2415 s1 = create_session_for_static_mapping_ed(sm, b1, l_key1,
2418 twice_nat1, is_lb1);
2422 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2428 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2434 s1 = pool_elt_at_index (tsm->sessions, value1.value);
2437 old_addr1 = ip1->dst_address.as_u32;
2438 new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
2439 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
2441 sum1 = ip1->checksum;
2442 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2444 if (PREDICT_FALSE (is_twice_nat_session (s1)))
2445 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2446 s1->ext_host_nat_addr.as_u32, ip4_header_t,
2448 ip1->checksum = ip_csum_fold (sum1);
2450 if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
2452 old_port1 = tcp1->dst_port;
2453 new_port1 = tcp1->dst_port = s1->in2out.port;
2455 sum1 = tcp1->checksum;
2456 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2458 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
2460 if (is_twice_nat_session (s1))
2462 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2463 s1->ext_host_nat_addr.as_u32,
2464 ip4_header_t, dst_address);
2465 sum1 = ip_csum_update (sum1, tcp1->src_port,
2466 s1->ext_host_nat_port, ip4_header_t,
2468 tcp1->src_port = s1->ext_host_nat_port;
2469 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2471 tcp1->checksum = ip_csum_fold(sum1);
2472 if (nat44_set_tcp_session_state_o2i (sm, s1, tcp1, thread_index))
2477 udp1->dst_port = s1->in2out.port;
2478 if (is_twice_nat_session (s1))
2480 udp1->src_port = s1->ext_host_nat_port;
2481 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2487 nat44_session_update_counters (s1, now,
2488 vlib_buffer_length_in_chain (vm, b1));
2489 /* Per-user LRU list maintenance */
2490 nat44_session_update_lru (sm, s1, thread_index);
2493 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2494 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2496 nat44_ed_out2in_trace_t *t =
2497 vlib_add_trace (vm, node, b1, sizeof (*t));
2498 t->is_slow_path = is_slow_path;
2499 t->sw_if_index = sw_if_index1;
2500 t->next_index = next1;
2501 t->session_index = ~0;
2503 t->session_index = s1 - tsm->sessions;
2506 pkts_processed += next1 != NAT44_ED_OUT2IN_NEXT_DROP;
2508 /* verify speculative enqueues, maybe switch current next frame */
2509 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2510 to_next, n_left_to_next,
2511 bi0, bi1, next0, next1);
2514 while (n_left_from > 0 && n_left_to_next > 0)
2518 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2519 u16 old_port0, new_port0;
2523 icmp46_header_t * icmp0;
2524 snat_session_t *s0 = 0;
2525 clib_bihash_kv_16_8_t kv0, value0;
2527 snat_session_key_t e_key0, l_key0;
2529 twice_nat_type_t twice_nat0;
2531 /* speculatively enqueue b0 to the current next frame */
2537 n_left_to_next -= 1;
2539 b0 = vlib_get_buffer (vm, bi0);
2540 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2541 vnet_buffer (b0)->snat.flags = 0;
2542 ip0 = vlib_buffer_get_current (b0);
2544 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2545 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2548 if (PREDICT_FALSE(ip0->ttl == 1))
2550 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2551 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2552 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2554 next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2558 udp0 = ip4_next_header (ip0);
2559 tcp0 = (tcp_header_t *) udp0;
2560 icmp0 = (icmp46_header_t *) udp0;
2561 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2565 if (PREDICT_FALSE (proto0 == ~0))
2567 s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2568 thread_index, now, vm, node);
2569 if (!sm->forwarding_enabled)
2572 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2577 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2579 next0 = icmp_out2in_slow_path
2580 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2581 next0, now, thread_index, &s0);
2587 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2589 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2593 if (ip4_is_fragment (ip0))
2595 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2596 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2601 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2602 rx_fib_index0, udp0->dst_port, udp0->src_port);
2604 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2608 /* Try to match static mapping by external address and port,
2609 destination address and port in packet */
2610 e_key0.addr = ip0->dst_address;
2611 e_key0.port = udp0->dst_port;
2612 e_key0.protocol = proto0;
2613 e_key0.fib_index = rx_fib_index0;
2614 if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2615 &twice_nat0, &is_lb0))
2618 * Send DHCP packets to the ipv4 stack, or we won't
2619 * be able to use dhcp client on the outside interface
2621 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2622 && (udp0->dst_port ==
2623 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2626 (vnet_buffer (b0)->sw_if_index[VLIB_RX], &next0, b0);
2630 if (!sm->forwarding_enabled)
2632 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2633 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2637 if (next_src_nat(sm, ip0, ip0->protocol,
2638 udp0->src_port, udp0->dst_port,
2641 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2644 create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2650 /* Create session initiated by host from external network */
2651 s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2654 twice_nat0, is_lb0);
2658 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2664 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2670 s0 = pool_elt_at_index (tsm->sessions, value0.value);
2673 old_addr0 = ip0->dst_address.as_u32;
2674 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2675 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2677 sum0 = ip0->checksum;
2678 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2680 if (PREDICT_FALSE (is_twice_nat_session (s0)))
2681 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2682 s0->ext_host_nat_addr.as_u32, ip4_header_t,
2684 ip0->checksum = ip_csum_fold (sum0);
2686 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2688 old_port0 = tcp0->dst_port;
2689 new_port0 = tcp0->dst_port = s0->in2out.port;
2691 sum0 = tcp0->checksum;
2692 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2694 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2696 if (is_twice_nat_session (s0))
2698 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2699 s0->ext_host_nat_addr.as_u32,
2700 ip4_header_t, dst_address);
2701 sum0 = ip_csum_update (sum0, tcp0->src_port,
2702 s0->ext_host_nat_port, ip4_header_t,
2704 tcp0->src_port = s0->ext_host_nat_port;
2705 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2707 tcp0->checksum = ip_csum_fold(sum0);
2708 if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2713 udp0->dst_port = s0->in2out.port;
2714 if (is_twice_nat_session (s0))
2716 udp0->src_port = s0->ext_host_nat_port;
2717 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2723 nat44_session_update_counters (s0, now,
2724 vlib_buffer_length_in_chain (vm, b0));
2725 /* Per-user LRU list maintenance */
2726 nat44_session_update_lru (sm, s0, thread_index);
2729 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2730 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2732 nat44_ed_out2in_trace_t *t =
2733 vlib_add_trace (vm, node, b0, sizeof (*t));
2734 t->is_slow_path = is_slow_path;
2735 t->sw_if_index = sw_if_index0;
2736 t->next_index = next0;
2737 t->session_index = ~0;
2739 t->session_index = s0 - tsm->sessions;
2742 pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2744 /* verify speculative enqueue, maybe switch current next frame */
2745 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2746 to_next, n_left_to_next,
2750 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2753 vlib_node_increment_counter (vm, stats_node_index,
2754 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2756 return frame->n_vectors;
2760 nat44_ed_out2in_fast_path_fn (vlib_main_t * vm,
2761 vlib_node_runtime_t * node,
2762 vlib_frame_t * frame)
2764 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
2767 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
2768 .function = nat44_ed_out2in_fast_path_fn,
2769 .name = "nat44-ed-out2in",
2770 .vector_size = sizeof (u32),
2771 .format_trace = format_nat44_ed_out2in_trace,
2772 .type = VLIB_NODE_TYPE_INTERNAL,
2774 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2775 .error_strings = snat_out2in_error_strings,
2777 .runtime_data_bytes = sizeof (snat_runtime_t),
2779 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2781 /* edit / add dispositions here */
2783 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2784 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2785 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2786 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2787 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2791 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_node, nat44_ed_out2in_fast_path_fn);
2794 nat44_ed_out2in_slow_path_fn (vlib_main_t * vm,
2795 vlib_node_runtime_t * node,
2796 vlib_frame_t * frame)
2798 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
2801 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
2802 .function = nat44_ed_out2in_slow_path_fn,
2803 .name = "nat44-ed-out2in-slowpath",
2804 .vector_size = sizeof (u32),
2805 .format_trace = format_nat44_ed_out2in_trace,
2806 .type = VLIB_NODE_TYPE_INTERNAL,
2808 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2809 .error_strings = snat_out2in_error_strings,
2811 .runtime_data_bytes = sizeof (snat_runtime_t),
2813 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2815 /* edit / add dispositions here */
2817 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2818 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2819 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2820 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2821 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2825 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_slowpath_node,
2826 nat44_ed_out2in_slow_path_fn);
2828 /**************************/
2829 /*** deterministic mode ***/
2830 /**************************/
2832 snat_det_out2in_node_fn (vlib_main_t * vm,
2833 vlib_node_runtime_t * node,
2834 vlib_frame_t * frame)
2836 u32 n_left_from, * from, * to_next;
2837 snat_out2in_next_t next_index;
2838 u32 pkts_processed = 0;
2839 snat_main_t * sm = &snat_main;
2840 u32 thread_index = vm->thread_index;
2842 from = vlib_frame_vector_args (frame);
2843 n_left_from = frame->n_vectors;
2844 next_index = node->cached_next_index;
2846 while (n_left_from > 0)
2850 vlib_get_next_frame (vm, node, next_index,
2851 to_next, n_left_to_next);
2853 while (n_left_from >= 4 && n_left_to_next >= 2)
2856 vlib_buffer_t * b0, * b1;
2857 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2858 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2859 u32 sw_if_index0, sw_if_index1;
2860 ip4_header_t * ip0, * ip1;
2861 ip_csum_t sum0, sum1;
2862 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2863 u16 new_port0, old_port0, old_port1, new_port1;
2864 udp_header_t * udp0, * udp1;
2865 tcp_header_t * tcp0, * tcp1;
2867 snat_det_out_key_t key0, key1;
2868 snat_det_map_t * dm0, * dm1;
2869 snat_det_session_t * ses0 = 0, * ses1 = 0;
2870 u32 rx_fib_index0, rx_fib_index1;
2871 icmp46_header_t * icmp0, * icmp1;
2873 /* Prefetch next iteration. */
2875 vlib_buffer_t * p2, * p3;
2877 p2 = vlib_get_buffer (vm, from[2]);
2878 p3 = vlib_get_buffer (vm, from[3]);
2880 vlib_prefetch_buffer_header (p2, LOAD);
2881 vlib_prefetch_buffer_header (p3, LOAD);
2883 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2884 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2887 /* speculatively enqueue b0 and b1 to the current next frame */
2888 to_next[0] = bi0 = from[0];
2889 to_next[1] = bi1 = from[1];
2893 n_left_to_next -= 2;
2895 b0 = vlib_get_buffer (vm, bi0);
2896 b1 = vlib_get_buffer (vm, bi1);
2898 ip0 = vlib_buffer_get_current (b0);
2899 udp0 = ip4_next_header (ip0);
2900 tcp0 = (tcp_header_t *) udp0;
2902 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2904 if (PREDICT_FALSE(ip0->ttl == 1))
2906 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2907 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2908 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2910 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2914 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2916 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2918 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2919 icmp0 = (icmp46_header_t *) udp0;
2921 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2922 rx_fib_index0, node, next0, thread_index,
2927 key0.ext_host_addr = ip0->src_address;
2928 key0.ext_host_port = tcp0->src;
2929 key0.out_port = tcp0->dst;
2931 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2932 if (PREDICT_FALSE(!dm0))
2934 nat_log_info ("unknown dst address: %U",
2935 format_ip4_address, &ip0->dst_address);
2936 next0 = SNAT_OUT2IN_NEXT_DROP;
2937 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2941 snat_det_reverse(dm0, &ip0->dst_address,
2942 clib_net_to_host_u16(tcp0->dst), &new_addr0);
2944 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2945 if (PREDICT_FALSE(!ses0))
2947 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2948 format_ip4_address, &ip0->src_address,
2949 clib_net_to_host_u16 (tcp0->src),
2950 format_ip4_address, &ip0->dst_address,
2951 clib_net_to_host_u16 (tcp0->dst),
2952 format_ip4_address, &new_addr0);
2953 next0 = SNAT_OUT2IN_NEXT_DROP;
2954 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2957 new_port0 = ses0->in_port;
2959 old_addr0 = ip0->dst_address;
2960 ip0->dst_address = new_addr0;
2961 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2963 sum0 = ip0->checksum;
2964 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2966 dst_address /* changed member */);
2967 ip0->checksum = ip_csum_fold (sum0);
2969 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2971 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2972 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2973 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2974 snat_det_ses_close(dm0, ses0);
2976 old_port0 = tcp0->dst;
2977 tcp0->dst = new_port0;
2979 sum0 = tcp0->checksum;
2980 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2982 dst_address /* changed member */);
2984 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2985 ip4_header_t /* cheat */,
2986 length /* changed member */);
2987 tcp0->checksum = ip_csum_fold(sum0);
2991 old_port0 = udp0->dst_port;
2992 udp0->dst_port = new_port0;
2998 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2999 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3001 snat_out2in_trace_t *t =
3002 vlib_add_trace (vm, node, b0, sizeof (*t));
3003 t->sw_if_index = sw_if_index0;
3004 t->next_index = next0;
3005 t->session_index = ~0;
3007 t->session_index = ses0 - dm0->sessions;
3010 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3012 b1 = vlib_get_buffer (vm, bi1);
3014 ip1 = vlib_buffer_get_current (b1);
3015 udp1 = ip4_next_header (ip1);
3016 tcp1 = (tcp_header_t *) udp1;
3018 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3020 if (PREDICT_FALSE(ip1->ttl == 1))
3022 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3023 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3024 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3026 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3030 proto1 = ip_proto_to_snat_proto (ip1->protocol);
3032 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3034 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3035 icmp1 = (icmp46_header_t *) udp1;
3037 next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
3038 rx_fib_index1, node, next1, thread_index,
3043 key1.ext_host_addr = ip1->src_address;
3044 key1.ext_host_port = tcp1->src;
3045 key1.out_port = tcp1->dst;
3047 dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
3048 if (PREDICT_FALSE(!dm1))
3050 nat_log_info ("unknown dst address: %U",
3051 format_ip4_address, &ip1->dst_address);
3052 next1 = SNAT_OUT2IN_NEXT_DROP;
3053 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3057 snat_det_reverse(dm1, &ip1->dst_address,
3058 clib_net_to_host_u16(tcp1->dst), &new_addr1);
3060 ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
3061 if (PREDICT_FALSE(!ses1))
3063 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3064 format_ip4_address, &ip1->src_address,
3065 clib_net_to_host_u16 (tcp1->src),
3066 format_ip4_address, &ip1->dst_address,
3067 clib_net_to_host_u16 (tcp1->dst),
3068 format_ip4_address, &new_addr1);
3069 next1 = SNAT_OUT2IN_NEXT_DROP;
3070 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3073 new_port1 = ses1->in_port;
3075 old_addr1 = ip1->dst_address;
3076 ip1->dst_address = new_addr1;
3077 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3079 sum1 = ip1->checksum;
3080 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3082 dst_address /* changed member */);
3083 ip1->checksum = ip_csum_fold (sum1);
3085 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3087 if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3088 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3089 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
3090 snat_det_ses_close(dm1, ses1);
3092 old_port1 = tcp1->dst;
3093 tcp1->dst = new_port1;
3095 sum1 = tcp1->checksum;
3096 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3098 dst_address /* changed member */);
3100 sum1 = ip_csum_update (sum1, old_port1, new_port1,
3101 ip4_header_t /* cheat */,
3102 length /* changed member */);
3103 tcp1->checksum = ip_csum_fold(sum1);
3107 old_port1 = udp1->dst_port;
3108 udp1->dst_port = new_port1;
3114 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3115 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3117 snat_out2in_trace_t *t =
3118 vlib_add_trace (vm, node, b1, sizeof (*t));
3119 t->sw_if_index = sw_if_index1;
3120 t->next_index = next1;
3121 t->session_index = ~0;
3123 t->session_index = ses1 - dm1->sessions;
3126 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
3128 /* verify speculative enqueues, maybe switch current next frame */
3129 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3130 to_next, n_left_to_next,
3131 bi0, bi1, next0, next1);
3134 while (n_left_from > 0 && n_left_to_next > 0)
3138 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
3142 ip4_address_t new_addr0, old_addr0;
3143 u16 new_port0, old_port0;
3144 udp_header_t * udp0;
3145 tcp_header_t * tcp0;
3147 snat_det_out_key_t key0;
3148 snat_det_map_t * dm0;
3149 snat_det_session_t * ses0 = 0;
3151 icmp46_header_t * icmp0;
3153 /* speculatively enqueue b0 to the current next frame */
3159 n_left_to_next -= 1;
3161 b0 = vlib_get_buffer (vm, bi0);
3163 ip0 = vlib_buffer_get_current (b0);
3164 udp0 = ip4_next_header (ip0);
3165 tcp0 = (tcp_header_t *) udp0;
3167 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3169 if (PREDICT_FALSE(ip0->ttl == 1))
3171 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3172 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3173 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3175 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3179 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3181 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3183 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3184 icmp0 = (icmp46_header_t *) udp0;
3186 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3187 rx_fib_index0, node, next0, thread_index,
3192 key0.ext_host_addr = ip0->src_address;
3193 key0.ext_host_port = tcp0->src;
3194 key0.out_port = tcp0->dst;
3196 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
3197 if (PREDICT_FALSE(!dm0))
3199 nat_log_info ("unknown dst address: %U",
3200 format_ip4_address, &ip0->dst_address);
3201 next0 = SNAT_OUT2IN_NEXT_DROP;
3202 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3206 snat_det_reverse(dm0, &ip0->dst_address,
3207 clib_net_to_host_u16(tcp0->dst), &new_addr0);
3209 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3210 if (PREDICT_FALSE(!ses0))
3212 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3213 format_ip4_address, &ip0->src_address,
3214 clib_net_to_host_u16 (tcp0->src),
3215 format_ip4_address, &ip0->dst_address,
3216 clib_net_to_host_u16 (tcp0->dst),
3217 format_ip4_address, &new_addr0);
3218 next0 = SNAT_OUT2IN_NEXT_DROP;
3219 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3222 new_port0 = ses0->in_port;
3224 old_addr0 = ip0->dst_address;
3225 ip0->dst_address = new_addr0;
3226 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3228 sum0 = ip0->checksum;
3229 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3231 dst_address /* changed member */);
3232 ip0->checksum = ip_csum_fold (sum0);
3234 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3236 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3237 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3238 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
3239 snat_det_ses_close(dm0, ses0);
3241 old_port0 = tcp0->dst;
3242 tcp0->dst = new_port0;
3244 sum0 = tcp0->checksum;
3245 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3247 dst_address /* changed member */);
3249 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3250 ip4_header_t /* cheat */,
3251 length /* changed member */);
3252 tcp0->checksum = ip_csum_fold(sum0);
3256 old_port0 = udp0->dst_port;
3257 udp0->dst_port = new_port0;
3263 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3264 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3266 snat_out2in_trace_t *t =
3267 vlib_add_trace (vm, node, b0, sizeof (*t));
3268 t->sw_if_index = sw_if_index0;
3269 t->next_index = next0;
3270 t->session_index = ~0;
3272 t->session_index = ses0 - dm0->sessions;
3275 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3277 /* verify speculative enqueue, maybe switch current next frame */
3278 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3279 to_next, n_left_to_next,
3283 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3286 vlib_node_increment_counter (vm, snat_det_out2in_node.index,
3287 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3289 return frame->n_vectors;
3292 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
3293 .function = snat_det_out2in_node_fn,
3294 .name = "nat44-det-out2in",
3295 .vector_size = sizeof (u32),
3296 .format_trace = format_snat_out2in_trace,
3297 .type = VLIB_NODE_TYPE_INTERNAL,
3299 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3300 .error_strings = snat_out2in_error_strings,
3302 .runtime_data_bytes = sizeof (snat_runtime_t),
3304 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3306 /* edit / add dispositions here */
3308 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3309 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3310 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3311 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3314 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
3317 * Get address and port values to be used for ICMP packet translation
3318 * and create session if needed
3320 * @param[in,out] sm NAT main
3321 * @param[in,out] node NAT node runtime
3322 * @param[in] thread_index thread index
3323 * @param[in,out] b0 buffer containing packet to be translated
3324 * @param[out] p_proto protocol used for matching
3325 * @param[out] p_value address and port after NAT translation
3326 * @param[out] p_dont_translate if packet should not be translated
3327 * @param d optional parameter
3328 * @param e optional parameter
3330 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
3331 u32 thread_index, vlib_buffer_t *b0,
3332 ip4_header_t *ip0, u8 *p_proto,
3333 snat_session_key_t *p_value,
3334 u8 *p_dont_translate, void *d, void *e)
3336 icmp46_header_t *icmp0;
3339 snat_det_out_key_t key0;
3340 u8 dont_translate = 0;
3342 icmp_echo_header_t *echo0, *inner_echo0 = 0;
3343 ip4_header_t *inner_ip0;
3344 void *l4_header = 0;
3345 icmp46_header_t *inner_icmp0;
3346 snat_det_map_t * dm0 = 0;
3347 ip4_address_t new_addr0 = {{0}};
3348 snat_det_session_t * ses0 = 0;
3349 ip4_address_t out_addr;
3351 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3352 echo0 = (icmp_echo_header_t *)(icmp0+1);
3353 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3355 if (!icmp_is_error_message (icmp0))
3357 protocol = SNAT_PROTOCOL_ICMP;
3358 key0.ext_host_addr = ip0->src_address;
3359 key0.ext_host_port = 0;
3360 key0.out_port = echo0->identifier;
3361 out_addr = ip0->dst_address;
3365 inner_ip0 = (ip4_header_t *)(echo0+1);
3366 l4_header = ip4_next_header (inner_ip0);
3367 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3368 key0.ext_host_addr = inner_ip0->dst_address;
3369 out_addr = inner_ip0->src_address;
3372 case SNAT_PROTOCOL_ICMP:
3373 inner_icmp0 = (icmp46_header_t*)l4_header;
3374 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3375 key0.ext_host_port = 0;
3376 key0.out_port = inner_echo0->identifier;
3378 case SNAT_PROTOCOL_UDP:
3379 case SNAT_PROTOCOL_TCP:
3380 key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3381 key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
3384 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
3385 next0 = SNAT_OUT2IN_NEXT_DROP;
3390 dm0 = snat_det_map_by_out(sm, &out_addr);
3391 if (PREDICT_FALSE(!dm0))
3393 /* Don't NAT packet aimed at the intfc address */
3394 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3395 ip0->dst_address.as_u32)))
3400 nat_log_info ("unknown dst address: %U",
3401 format_ip4_address, &ip0->dst_address);
3405 snat_det_reverse(dm0, &ip0->dst_address,
3406 clib_net_to_host_u16(key0.out_port), &new_addr0);
3408 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3409 if (PREDICT_FALSE(!ses0))
3411 /* Don't NAT packet aimed at the intfc address */
3412 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3413 ip0->dst_address.as_u32)))
3418 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3419 format_ip4_address, &key0.ext_host_addr,
3420 clib_net_to_host_u16 (key0.ext_host_port),
3421 format_ip4_address, &out_addr,
3422 clib_net_to_host_u16 (key0.out_port),
3423 format_ip4_address, &new_addr0);
3424 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3425 next0 = SNAT_OUT2IN_NEXT_DROP;
3429 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
3430 !icmp_is_error_message (icmp0)))
3432 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
3433 next0 = SNAT_OUT2IN_NEXT_DROP;
3440 *p_proto = protocol;
3443 p_value->addr = new_addr0;
3444 p_value->fib_index = sm->inside_fib_index;
3445 p_value->port = ses0->in_port;
3447 *p_dont_translate = dont_translate;
3449 *(snat_det_session_t**)d = ses0;
3451 *(snat_det_map_t**)e = dm0;
3455 /**********************/
3456 /*** worker handoff ***/
3457 /**********************/
3459 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
3460 vlib_node_runtime_t * node,
3461 vlib_frame_t * frame)
3463 snat_main_t *sm = &snat_main;
3464 vlib_thread_main_t *tm = vlib_get_thread_main ();
3465 u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
3466 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3467 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3469 vlib_frame_queue_elt_t *hf = 0;
3470 vlib_frame_queue_t *fq;
3471 vlib_frame_t *f = 0;
3473 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3474 u32 next_worker_index = 0;
3475 u32 current_worker_index = ~0;
3476 u32 thread_index = vm->thread_index;
3477 vlib_frame_t *d = 0;
3479 ASSERT (vec_len (sm->workers));
3481 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3483 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3485 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3486 tm->n_vlib_mains - 1,
3487 (vlib_frame_queue_t *) (~0));
3490 from = vlib_frame_vector_args (frame);
3491 n_left_from = frame->n_vectors;
3493 while (n_left_from > 0)
3506 b0 = vlib_get_buffer (vm, bi0);
3508 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3509 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3511 ip0 = vlib_buffer_get_current (b0);
3513 next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
3515 if (PREDICT_FALSE (next_worker_index != thread_index))
3519 if (next_worker_index != current_worker_index)
3521 fq = is_vlib_frame_queue_congested (
3522 sm->fq_out2in_index, next_worker_index, NAT_FQ_NELTS - 2,
3523 congested_handoff_queue_by_worker_index);
3527 /* if this is 1st frame */
3530 d = vlib_get_frame_to_node (vm, sm->error_node_index);
3531 to_next_drop = vlib_frame_vector_args (d);
3534 to_next_drop[0] = bi0;
3537 b0->error = node->errors[SNAT_OUT2IN_ERROR_FQ_CONGESTED];
3542 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3544 hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
3546 handoff_queue_elt_by_worker_index);
3548 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3549 to_next_worker = &hf->buffer_index[hf->n_vectors];
3550 current_worker_index = next_worker_index;
3553 /* enqueue to correct worker thread */
3554 to_next_worker[0] = bi0;
3556 n_left_to_next_worker--;
3558 if (n_left_to_next_worker == 0)
3560 hf->n_vectors = VLIB_FRAME_SIZE;
3561 vlib_put_frame_queue_elt (hf);
3562 current_worker_index = ~0;
3563 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3570 /* if this is 1st frame */
3573 f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
3574 to_next = vlib_frame_vector_args (f);
3583 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3584 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3586 snat_out2in_worker_handoff_trace_t *t =
3587 vlib_add_trace (vm, node, b0, sizeof (*t));
3588 t->next_worker_index = next_worker_index;
3589 t->do_handoff = do_handoff;
3594 vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
3597 vlib_put_frame_to_node (vm, sm->error_node_index, d);
3600 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3602 /* Ship frames to the worker nodes */
3603 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3605 if (handoff_queue_elt_by_worker_index[i])
3607 hf = handoff_queue_elt_by_worker_index[i];
3609 * It works better to let the handoff node
3610 * rate-adapt, always ship the handoff queue element.
3612 if (1 || hf->n_vectors == hf->last_n_vectors)
3614 vlib_put_frame_queue_elt (hf);
3615 handoff_queue_elt_by_worker_index[i] = 0;
3618 hf->last_n_vectors = hf->n_vectors;
3620 congested_handoff_queue_by_worker_index[i] =
3621 (vlib_frame_queue_t *) (~0);
3624 current_worker_index = ~0;
3625 return frame->n_vectors;
3628 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
3629 .function = snat_out2in_worker_handoff_fn,
3630 .name = "nat44-out2in-worker-handoff",
3631 .vector_size = sizeof (u32),
3632 .format_trace = format_snat_out2in_worker_handoff_trace,
3633 .type = VLIB_NODE_TYPE_INTERNAL,
3635 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3636 .error_strings = snat_out2in_error_strings,
3645 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
3648 snat_out2in_fast_node_fn (vlib_main_t * vm,
3649 vlib_node_runtime_t * node,
3650 vlib_frame_t * frame)
3652 u32 n_left_from, * from, * to_next;
3653 snat_out2in_next_t next_index;
3654 u32 pkts_processed = 0;
3655 snat_main_t * sm = &snat_main;
3657 from = vlib_frame_vector_args (frame);
3658 n_left_from = frame->n_vectors;
3659 next_index = node->cached_next_index;
3661 while (n_left_from > 0)
3665 vlib_get_next_frame (vm, node, next_index,
3666 to_next, n_left_to_next);
3668 while (n_left_from > 0 && n_left_to_next > 0)
3672 u32 next0 = SNAT_OUT2IN_NEXT_DROP;
3676 u32 new_addr0, old_addr0;
3677 u16 new_port0, old_port0;
3678 udp_header_t * udp0;
3679 tcp_header_t * tcp0;
3680 icmp46_header_t * icmp0;
3681 snat_session_key_t key0, sm0;
3685 /* speculatively enqueue b0 to the current next frame */
3691 n_left_to_next -= 1;
3693 b0 = vlib_get_buffer (vm, bi0);
3695 ip0 = vlib_buffer_get_current (b0);
3696 udp0 = ip4_next_header (ip0);
3697 tcp0 = (tcp_header_t *) udp0;
3698 icmp0 = (icmp46_header_t *) udp0;
3700 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3701 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3703 vnet_feature_next (sw_if_index0, &next0, b0);
3705 if (PREDICT_FALSE(ip0->ttl == 1))
3707 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3708 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3709 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3711 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3715 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3717 if (PREDICT_FALSE (proto0 == ~0))
3720 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3722 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3723 rx_fib_index0, node, next0, ~0, 0, 0);
3727 key0.addr = ip0->dst_address;
3728 key0.port = udp0->dst_port;
3729 key0.fib_index = rx_fib_index0;
3731 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
3733 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3737 new_addr0 = sm0.addr.as_u32;
3738 new_port0 = sm0.port;
3739 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3740 old_addr0 = ip0->dst_address.as_u32;
3741 ip0->dst_address.as_u32 = new_addr0;
3743 sum0 = ip0->checksum;
3744 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3746 dst_address /* changed member */);
3747 ip0->checksum = ip_csum_fold (sum0);
3749 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3751 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3753 old_port0 = tcp0->dst_port;
3754 tcp0->dst_port = new_port0;
3756 sum0 = tcp0->checksum;
3757 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3759 dst_address /* changed member */);
3761 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3762 ip4_header_t /* cheat */,
3763 length /* changed member */);
3764 tcp0->checksum = ip_csum_fold(sum0);
3768 old_port0 = udp0->dst_port;
3769 udp0->dst_port = new_port0;
3775 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3777 sum0 = tcp0->checksum;
3778 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3780 dst_address /* changed member */);
3782 tcp0->checksum = ip_csum_fold(sum0);
3788 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3789 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3791 snat_out2in_trace_t *t =
3792 vlib_add_trace (vm, node, b0, sizeof (*t));
3793 t->sw_if_index = sw_if_index0;
3794 t->next_index = next0;
3797 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3799 /* verify speculative enqueue, maybe switch current next frame */
3800 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3801 to_next, n_left_to_next,
3805 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3808 vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3809 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3811 return frame->n_vectors;
3814 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3815 .function = snat_out2in_fast_node_fn,
3816 .name = "nat44-out2in-fast",
3817 .vector_size = sizeof (u32),
3818 .format_trace = format_snat_out2in_fast_trace,
3819 .type = VLIB_NODE_TYPE_INTERNAL,
3821 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3822 .error_strings = snat_out2in_error_strings,
3824 .runtime_data_bytes = sizeof (snat_runtime_t),
3826 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3828 /* edit / add dispositions here */
3830 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3831 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3832 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3833 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3836 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);