2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
26 #include <nat/nat_ipfix_logging.h>
27 #include <nat/nat_det.h>
28 #include <nat/nat_reass.h>
29 #include <nat/nat_inlines.h>
31 #include <vppinfra/hash.h>
32 #include <vppinfra/error.h>
33 #include <vppinfra/elog.h>
39 } snat_out2in_trace_t;
42 u32 next_worker_index;
44 } snat_out2in_worker_handoff_trace_t;
46 /* packet trace format function */
47 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
49 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
50 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
51 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
53 s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
54 t->sw_if_index, t->next_index, t->session_index);
58 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
60 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
61 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
62 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
64 s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
65 t->sw_if_index, t->next_index);
69 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
71 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
72 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
73 snat_out2in_worker_handoff_trace_t * t =
74 va_arg (*args, snat_out2in_worker_handoff_trace_t *);
77 m = t->do_handoff ? "next worker" : "same worker";
78 s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
87 } nat44_out2in_reass_trace_t;
89 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
91 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
92 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
93 nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
95 s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
96 t->sw_if_index, t->next_index,
97 t->cached ? "cached" : "translated");
102 vlib_node_registration_t snat_out2in_node;
103 vlib_node_registration_t snat_out2in_fast_node;
104 vlib_node_registration_t snat_out2in_worker_handoff_node;
105 vlib_node_registration_t snat_det_out2in_node;
106 vlib_node_registration_t nat44_out2in_reass_node;
107 vlib_node_registration_t nat44_ed_out2in_node;
108 vlib_node_registration_t nat44_ed_out2in_slowpath_node;
110 #define foreach_snat_out2in_error \
111 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
112 _(OUT2IN_PACKETS, "Good out2in packets processed") \
113 _(OUT_OF_PORTS, "Out of ports") \
114 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
115 _(NO_TRANSLATION, "No translation") \
116 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
117 _(DROP_FRAGMENT, "Drop fragment") \
118 _(MAX_REASS, "Maximum reassemblies exceeded") \
119 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
120 _(FQ_CONGESTED, "Handoff frame queue congested")
123 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
124 foreach_snat_out2in_error
127 } snat_out2in_error_t;
129 static char * snat_out2in_error_strings[] = {
130 #define _(sym,string) string,
131 foreach_snat_out2in_error
136 SNAT_OUT2IN_NEXT_DROP,
137 SNAT_OUT2IN_NEXT_LOOKUP,
138 SNAT_OUT2IN_NEXT_ICMP_ERROR,
139 SNAT_OUT2IN_NEXT_REASS,
141 } snat_out2in_next_t;
144 * @brief Create session for static mapping.
146 * Create NAT session initiated by host from external network with static
149 * @param sm NAT main.
150 * @param b0 Vlib buffer.
151 * @param in2out In2out NAT44 session key.
152 * @param out2in Out2in NAT44 session key.
153 * @param node Vlib node.
155 * @returns SNAT session if successfully created otherwise 0.
157 static inline snat_session_t *
158 create_session_for_static_mapping (snat_main_t *sm,
160 snat_session_key_t in2out,
161 snat_session_key_t out2in,
162 vlib_node_runtime_t * node,
167 clib_bihash_kv_8_8_t kv0;
171 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
173 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
174 nat_log_notice ("maximum sessions exceeded");
178 ip0 = vlib_buffer_get_current (b0);
179 udp0 = ip4_next_header (ip0);
181 u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
184 nat_log_warn ("create NAT user failed");
188 s = nat_session_alloc_or_recycle (sm, u, thread_index);
191 nat_log_warn ("create NAT session failed");
195 s->outside_address_index = ~0;
196 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
197 s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
198 s->ext_host_port = udp0->src_port;
199 user_session_increment (sm, u, 1 /* static */);
202 s->in2out.protocol = out2in.protocol;
204 /* Add to translation hashes */
205 kv0.key = s->in2out.as_u64;
206 kv0.value = s - sm->per_thread_data[thread_index].sessions;
207 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
209 nat_log_notice ("in2out key add failed");
211 kv0.key = s->out2in.as_u64;
213 if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
215 nat_log_notice ("out2in key add failed");
218 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
219 s->out2in.addr.as_u32,
223 s->in2out.fib_index);
228 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
229 snat_session_key_t *p_key0)
231 icmp46_header_t *icmp0;
232 snat_session_key_t key0;
233 icmp_echo_header_t *echo0, *inner_echo0 = 0;
234 ip4_header_t *inner_ip0;
236 icmp46_header_t *inner_icmp0;
238 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
239 echo0 = (icmp_echo_header_t *)(icmp0+1);
241 if (!icmp_is_error_message (icmp0))
243 key0.protocol = SNAT_PROTOCOL_ICMP;
244 key0.addr = ip0->dst_address;
245 key0.port = echo0->identifier;
249 inner_ip0 = (ip4_header_t *)(echo0+1);
250 l4_header = ip4_next_header (inner_ip0);
251 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
252 key0.addr = inner_ip0->src_address;
253 switch (key0.protocol)
255 case SNAT_PROTOCOL_ICMP:
256 inner_icmp0 = (icmp46_header_t*)l4_header;
257 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
258 key0.port = inner_echo0->identifier;
260 case SNAT_PROTOCOL_UDP:
261 case SNAT_PROTOCOL_TCP:
262 key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
265 return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
269 return -1; /* success */
273 * Get address and port values to be used for ICMP packet translation
274 * and create session if needed
276 * @param[in,out] sm NAT main
277 * @param[in,out] node NAT node runtime
278 * @param[in] thread_index thread index
279 * @param[in,out] b0 buffer containing packet to be translated
280 * @param[out] p_proto protocol used for matching
281 * @param[out] p_value address and port after NAT translation
282 * @param[out] p_dont_translate if packet should not be translated
283 * @param d optional parameter
284 * @param e optional parameter
286 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
287 u32 thread_index, vlib_buffer_t *b0,
288 ip4_header_t *ip0, u8 *p_proto,
289 snat_session_key_t *p_value,
290 u8 *p_dont_translate, void *d, void *e)
292 icmp46_header_t *icmp0;
295 snat_session_key_t key0;
296 snat_session_key_t sm0;
297 snat_session_t *s0 = 0;
298 u8 dont_translate = 0;
299 clib_bihash_kv_8_8_t kv0, value0;
304 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
305 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
306 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
310 err = icmp_get_key (ip0, &key0);
313 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
314 next0 = SNAT_OUT2IN_NEXT_DROP;
317 key0.fib_index = rx_fib_index0;
319 kv0.key = key0.as_u64;
321 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
324 /* Try to match static mapping by external address and port,
325 destination address and port in packet */
326 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
328 if (!sm->forwarding_enabled)
330 /* Don't NAT packet aimed at the intfc address */
331 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
332 ip0->dst_address.as_u32)))
337 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
338 next0 = SNAT_OUT2IN_NEXT_DROP;
348 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
349 (icmp0->type != ICMP4_echo_request || !is_addr_only)))
351 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
352 next0 = SNAT_OUT2IN_NEXT_DROP;
356 /* Create session initiated by host from external network */
357 s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
362 next0 = SNAT_OUT2IN_NEXT_DROP;
368 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
369 icmp0->type != ICMP4_echo_request &&
370 !icmp_is_error_message (icmp0)))
372 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
373 next0 = SNAT_OUT2IN_NEXT_DROP;
377 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
382 *p_proto = key0.protocol;
384 *p_value = s0->in2out;
385 *p_dont_translate = dont_translate;
387 *(snat_session_t**)d = s0;
392 * Get address and port values to be used for ICMP packet translation
394 * @param[in] sm NAT main
395 * @param[in,out] node NAT node runtime
396 * @param[in] thread_index thread index
397 * @param[in,out] b0 buffer containing packet to be translated
398 * @param[out] p_proto protocol used for matching
399 * @param[out] p_value address and port after NAT translation
400 * @param[out] p_dont_translate if packet should not be translated
401 * @param d optional parameter
402 * @param e optional parameter
404 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
405 u32 thread_index, vlib_buffer_t *b0,
406 ip4_header_t *ip0, u8 *p_proto,
407 snat_session_key_t *p_value,
408 u8 *p_dont_translate, void *d, void *e)
410 icmp46_header_t *icmp0;
413 snat_session_key_t key0;
414 snat_session_key_t sm0;
415 u8 dont_translate = 0;
420 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
421 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
422 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
424 err = icmp_get_key (ip0, &key0);
427 b0->error = node->errors[err];
428 next0 = SNAT_OUT2IN_NEXT_DROP;
431 key0.fib_index = rx_fib_index0;
433 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
435 /* Don't NAT packet aimed at the intfc address */
436 if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
441 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
442 next0 = SNAT_OUT2IN_NEXT_DROP;
446 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
447 (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
448 !icmp_is_error_message (icmp0)))
450 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
451 next0 = SNAT_OUT2IN_NEXT_DROP;
458 *p_proto = key0.protocol;
459 *p_dont_translate = dont_translate;
463 static inline u32 icmp_out2in (snat_main_t *sm,
466 icmp46_header_t * icmp0,
469 vlib_node_runtime_t * node,
475 snat_session_key_t sm0;
477 icmp_echo_header_t *echo0, *inner_echo0 = 0;
478 ip4_header_t *inner_ip0 = 0;
480 icmp46_header_t *inner_icmp0;
482 u32 new_addr0, old_addr0;
483 u16 old_id0, new_id0;
488 echo0 = (icmp_echo_header_t *)(icmp0+1);
490 next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
491 &protocol, &sm0, &dont_translate, d, e);
494 if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
497 sum0 = ip_incremental_checksum (0, icmp0,
498 ntohs(ip0->length) - ip4_header_bytes (ip0));
499 checksum0 = ~ip_csum_fold (sum0);
500 if (checksum0 != 0 && checksum0 != 0xffff)
502 next0 = SNAT_OUT2IN_NEXT_DROP;
506 old_addr0 = ip0->dst_address.as_u32;
507 new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
508 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
510 sum0 = ip0->checksum;
511 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
512 dst_address /* changed member */);
513 ip0->checksum = ip_csum_fold (sum0);
515 if (icmp0->checksum == 0)
516 icmp0->checksum = 0xffff;
518 if (!icmp_is_error_message (icmp0))
521 if (PREDICT_FALSE(new_id0 != echo0->identifier))
523 old_id0 = echo0->identifier;
525 echo0->identifier = new_id0;
527 sum0 = icmp0->checksum;
528 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
529 identifier /* changed member */);
530 icmp0->checksum = ip_csum_fold (sum0);
535 inner_ip0 = (ip4_header_t *)(echo0+1);
536 l4_header = ip4_next_header (inner_ip0);
538 if (!ip4_header_checksum_is_valid (inner_ip0))
540 next0 = SNAT_OUT2IN_NEXT_DROP;
544 old_addr0 = inner_ip0->src_address.as_u32;
545 inner_ip0->src_address = sm0.addr;
546 new_addr0 = inner_ip0->src_address.as_u32;
548 sum0 = icmp0->checksum;
549 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
550 src_address /* changed member */);
551 icmp0->checksum = ip_csum_fold (sum0);
555 case SNAT_PROTOCOL_ICMP:
556 inner_icmp0 = (icmp46_header_t*)l4_header;
557 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
559 old_id0 = inner_echo0->identifier;
561 inner_echo0->identifier = new_id0;
563 sum0 = icmp0->checksum;
564 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
566 icmp0->checksum = ip_csum_fold (sum0);
568 case SNAT_PROTOCOL_UDP:
569 case SNAT_PROTOCOL_TCP:
570 old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
572 ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
574 sum0 = icmp0->checksum;
575 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
577 icmp0->checksum = ip_csum_fold (sum0);
589 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
592 icmp46_header_t * icmp0,
595 vlib_node_runtime_t * node,
598 snat_session_t ** p_s0)
600 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
601 next0, thread_index, p_s0, 0);
602 snat_session_t * s0 = *p_s0;
603 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
606 nat44_session_update_counters (s0, now,
607 vlib_buffer_length_in_chain (sm->vlib_main, b0));
608 /* Per-user LRU list maintenance */
609 nat44_session_update_lru (sm, s0, thread_index);
615 nat_out2in_sm_unknown_proto (snat_main_t *sm,
620 clib_bihash_kv_8_8_t kv, value;
621 snat_static_mapping_t *m;
622 snat_session_key_t m_key;
623 u32 old_addr, new_addr;
626 m_key.addr = ip->dst_address;
630 kv.key = m_key.as_u64;
631 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
634 m = pool_elt_at_index (sm->static_mappings, value.value);
636 old_addr = ip->dst_address.as_u32;
637 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
639 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
640 ip->checksum = ip_csum_fold (sum);
642 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
647 snat_out2in_node_fn (vlib_main_t * vm,
648 vlib_node_runtime_t * node,
649 vlib_frame_t * frame)
651 u32 n_left_from, * from, * to_next;
652 snat_out2in_next_t next_index;
653 u32 pkts_processed = 0;
654 snat_main_t * sm = &snat_main;
655 f64 now = vlib_time_now (vm);
656 u32 thread_index = vm->thread_index;
658 from = vlib_frame_vector_args (frame);
659 n_left_from = frame->n_vectors;
660 next_index = node->cached_next_index;
662 while (n_left_from > 0)
666 vlib_get_next_frame (vm, node, next_index,
667 to_next, n_left_to_next);
669 while (n_left_from >= 4 && n_left_to_next >= 2)
672 vlib_buffer_t * b0, * b1;
673 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
674 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
675 u32 sw_if_index0, sw_if_index1;
676 ip4_header_t * ip0, *ip1;
677 ip_csum_t sum0, sum1;
678 u32 new_addr0, old_addr0;
679 u16 new_port0, old_port0;
680 u32 new_addr1, old_addr1;
681 u16 new_port1, old_port1;
682 udp_header_t * udp0, * udp1;
683 tcp_header_t * tcp0, * tcp1;
684 icmp46_header_t * icmp0, * icmp1;
685 snat_session_key_t key0, key1, sm0, sm1;
686 u32 rx_fib_index0, rx_fib_index1;
688 snat_session_t * s0 = 0, * s1 = 0;
689 clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
691 /* Prefetch next iteration. */
693 vlib_buffer_t * p2, * p3;
695 p2 = vlib_get_buffer (vm, from[2]);
696 p3 = vlib_get_buffer (vm, from[3]);
698 vlib_prefetch_buffer_header (p2, LOAD);
699 vlib_prefetch_buffer_header (p3, LOAD);
701 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
702 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
705 /* speculatively enqueue b0 and b1 to the current next frame */
706 to_next[0] = bi0 = from[0];
707 to_next[1] = bi1 = from[1];
713 b0 = vlib_get_buffer (vm, bi0);
714 b1 = vlib_get_buffer (vm, bi1);
716 vnet_buffer (b0)->snat.flags = 0;
717 vnet_buffer (b1)->snat.flags = 0;
719 ip0 = vlib_buffer_get_current (b0);
720 udp0 = ip4_next_header (ip0);
721 tcp0 = (tcp_header_t *) udp0;
722 icmp0 = (icmp46_header_t *) udp0;
724 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
725 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
728 if (PREDICT_FALSE(ip0->ttl == 1))
730 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
731 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
732 ICMP4_time_exceeded_ttl_exceeded_in_transit,
734 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
738 proto0 = ip_proto_to_snat_proto (ip0->protocol);
740 if (PREDICT_FALSE (proto0 == ~0))
742 if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
744 if (!sm->forwarding_enabled)
746 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
747 next0 = SNAT_OUT2IN_NEXT_DROP;
753 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
755 next0 = icmp_out2in_slow_path
756 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
757 next0, now, thread_index, &s0);
761 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
763 next0 = SNAT_OUT2IN_NEXT_REASS;
767 key0.addr = ip0->dst_address;
768 key0.port = udp0->dst_port;
769 key0.protocol = proto0;
770 key0.fib_index = rx_fib_index0;
772 kv0.key = key0.as_u64;
774 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
777 /* Try to match static mapping by external address and port,
778 destination address and port in packet */
779 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
782 * Send DHCP packets to the ipv4 stack, or we won't
783 * be able to use dhcp client on the outside interface
785 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
786 && (udp0->dst_port ==
787 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
789 vnet_feature_next (&next0, b0);
793 if (!sm->forwarding_enabled)
795 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
796 next0 = SNAT_OUT2IN_NEXT_DROP;
801 /* Create session initiated by host from external network */
802 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
806 next0 = SNAT_OUT2IN_NEXT_DROP;
811 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
814 old_addr0 = ip0->dst_address.as_u32;
815 ip0->dst_address = s0->in2out.addr;
816 new_addr0 = ip0->dst_address.as_u32;
817 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
819 sum0 = ip0->checksum;
820 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
822 dst_address /* changed member */);
823 ip0->checksum = ip_csum_fold (sum0);
825 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
827 old_port0 = tcp0->dst_port;
828 tcp0->dst_port = s0->in2out.port;
829 new_port0 = tcp0->dst_port;
831 sum0 = tcp0->checksum;
832 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
834 dst_address /* changed member */);
836 sum0 = ip_csum_update (sum0, old_port0, new_port0,
837 ip4_header_t /* cheat */,
838 length /* changed member */);
839 tcp0->checksum = ip_csum_fold(sum0);
843 old_port0 = udp0->dst_port;
844 udp0->dst_port = s0->in2out.port;
849 nat44_session_update_counters (s0, now,
850 vlib_buffer_length_in_chain (vm, b0));
851 /* Per-user LRU list maintenance */
852 nat44_session_update_lru (sm, s0, thread_index);
855 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
856 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
858 snat_out2in_trace_t *t =
859 vlib_add_trace (vm, node, b0, sizeof (*t));
860 t->sw_if_index = sw_if_index0;
861 t->next_index = next0;
862 t->session_index = ~0;
864 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
867 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
870 ip1 = vlib_buffer_get_current (b1);
871 udp1 = ip4_next_header (ip1);
872 tcp1 = (tcp_header_t *) udp1;
873 icmp1 = (icmp46_header_t *) udp1;
875 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
876 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
879 if (PREDICT_FALSE(ip1->ttl == 1))
881 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
882 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
883 ICMP4_time_exceeded_ttl_exceeded_in_transit,
885 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
889 proto1 = ip_proto_to_snat_proto (ip1->protocol);
891 if (PREDICT_FALSE (proto1 == ~0))
893 if (nat_out2in_sm_unknown_proto(sm, b1, ip1, rx_fib_index1))
895 if (!sm->forwarding_enabled)
897 b1->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
898 next1 = SNAT_OUT2IN_NEXT_DROP;
904 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
906 next1 = icmp_out2in_slow_path
907 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
908 next1, now, thread_index, &s1);
912 if (PREDICT_FALSE (ip4_is_fragment (ip1)))
914 next1 = SNAT_OUT2IN_NEXT_REASS;
918 key1.addr = ip1->dst_address;
919 key1.port = udp1->dst_port;
920 key1.protocol = proto1;
921 key1.fib_index = rx_fib_index1;
923 kv1.key = key1.as_u64;
925 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
928 /* Try to match static mapping by external address and port,
929 destination address and port in packet */
930 if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
933 * Send DHCP packets to the ipv4 stack, or we won't
934 * be able to use dhcp client on the outside interface
936 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
937 && (udp1->dst_port ==
938 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
940 vnet_feature_next (&next1, b1);
944 if (!sm->forwarding_enabled)
946 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
947 next1 = SNAT_OUT2IN_NEXT_DROP;
952 /* Create session initiated by host from external network */
953 s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
957 next1 = SNAT_OUT2IN_NEXT_DROP;
962 s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
965 old_addr1 = ip1->dst_address.as_u32;
966 ip1->dst_address = s1->in2out.addr;
967 new_addr1 = ip1->dst_address.as_u32;
968 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
970 sum1 = ip1->checksum;
971 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
973 dst_address /* changed member */);
974 ip1->checksum = ip_csum_fold (sum1);
976 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
978 old_port1 = tcp1->dst_port;
979 tcp1->dst_port = s1->in2out.port;
980 new_port1 = tcp1->dst_port;
982 sum1 = tcp1->checksum;
983 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
985 dst_address /* changed member */);
987 sum1 = ip_csum_update (sum1, old_port1, new_port1,
988 ip4_header_t /* cheat */,
989 length /* changed member */);
990 tcp1->checksum = ip_csum_fold(sum1);
994 old_port1 = udp1->dst_port;
995 udp1->dst_port = s1->in2out.port;
1000 nat44_session_update_counters (s1, now,
1001 vlib_buffer_length_in_chain (vm, b1));
1002 /* Per-user LRU list maintenance */
1003 nat44_session_update_lru (sm, s1, thread_index);
1006 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1007 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1009 snat_out2in_trace_t *t =
1010 vlib_add_trace (vm, node, b1, sizeof (*t));
1011 t->sw_if_index = sw_if_index1;
1012 t->next_index = next1;
1013 t->session_index = ~0;
1015 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1018 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1020 /* verify speculative enqueues, maybe switch current next frame */
1021 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1022 to_next, n_left_to_next,
1023 bi0, bi1, next0, next1);
1026 while (n_left_from > 0 && n_left_to_next > 0)
1030 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1034 u32 new_addr0, old_addr0;
1035 u16 new_port0, old_port0;
1036 udp_header_t * udp0;
1037 tcp_header_t * tcp0;
1038 icmp46_header_t * icmp0;
1039 snat_session_key_t key0, sm0;
1042 snat_session_t * s0 = 0;
1043 clib_bihash_kv_8_8_t kv0, value0;
1045 /* speculatively enqueue b0 to the current next frame */
1051 n_left_to_next -= 1;
1053 b0 = vlib_get_buffer (vm, bi0);
1055 vnet_buffer (b0)->snat.flags = 0;
1057 ip0 = vlib_buffer_get_current (b0);
1058 udp0 = ip4_next_header (ip0);
1059 tcp0 = (tcp_header_t *) udp0;
1060 icmp0 = (icmp46_header_t *) udp0;
1062 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1063 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1066 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1068 if (PREDICT_FALSE (proto0 == ~0))
1070 if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
1072 if (!sm->forwarding_enabled)
1074 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1075 next0 = SNAT_OUT2IN_NEXT_DROP;
1081 if (PREDICT_FALSE(ip0->ttl == 1))
1083 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1084 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1085 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1087 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1091 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1093 next0 = icmp_out2in_slow_path
1094 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1095 next0, now, thread_index, &s0);
1099 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1101 next0 = SNAT_OUT2IN_NEXT_REASS;
1105 key0.addr = ip0->dst_address;
1106 key0.port = udp0->dst_port;
1107 key0.protocol = proto0;
1108 key0.fib_index = rx_fib_index0;
1110 kv0.key = key0.as_u64;
1112 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1115 /* Try to match static mapping by external address and port,
1116 destination address and port in packet */
1117 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1120 * Send DHCP packets to the ipv4 stack, or we won't
1121 * be able to use dhcp client on the outside interface
1123 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1124 && (udp0->dst_port ==
1125 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1127 vnet_feature_next (&next0, b0);
1131 if (!sm->forwarding_enabled)
1133 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1134 next0 = SNAT_OUT2IN_NEXT_DROP;
1139 /* Create session initiated by host from external network */
1140 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1144 next0 = SNAT_OUT2IN_NEXT_DROP;
1149 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1152 old_addr0 = ip0->dst_address.as_u32;
1153 ip0->dst_address = s0->in2out.addr;
1154 new_addr0 = ip0->dst_address.as_u32;
1155 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1157 sum0 = ip0->checksum;
1158 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1160 dst_address /* changed member */);
1161 ip0->checksum = ip_csum_fold (sum0);
1163 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1165 old_port0 = tcp0->dst_port;
1166 tcp0->dst_port = s0->in2out.port;
1167 new_port0 = tcp0->dst_port;
1169 sum0 = tcp0->checksum;
1170 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1172 dst_address /* changed member */);
1174 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1175 ip4_header_t /* cheat */,
1176 length /* changed member */);
1177 tcp0->checksum = ip_csum_fold(sum0);
1181 old_port0 = udp0->dst_port;
1182 udp0->dst_port = s0->in2out.port;
1187 nat44_session_update_counters (s0, now,
1188 vlib_buffer_length_in_chain (vm, b0));
1189 /* Per-user LRU list maintenance */
1190 nat44_session_update_lru (sm, s0, thread_index);
1193 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1194 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1196 snat_out2in_trace_t *t =
1197 vlib_add_trace (vm, node, b0, sizeof (*t));
1198 t->sw_if_index = sw_if_index0;
1199 t->next_index = next0;
1200 t->session_index = ~0;
1202 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1205 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1207 /* verify speculative enqueue, maybe switch current next frame */
1208 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1209 to_next, n_left_to_next,
1213 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1216 vlib_node_increment_counter (vm, snat_out2in_node.index,
1217 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1219 return frame->n_vectors;
1222 VLIB_REGISTER_NODE (snat_out2in_node) = {
1223 .function = snat_out2in_node_fn,
1224 .name = "nat44-out2in",
1225 .vector_size = sizeof (u32),
1226 .format_trace = format_snat_out2in_trace,
1227 .type = VLIB_NODE_TYPE_INTERNAL,
1229 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1230 .error_strings = snat_out2in_error_strings,
1232 .runtime_data_bytes = sizeof (snat_runtime_t),
1234 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1236 /* edit / add dispositions here */
1238 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1239 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1240 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1241 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1244 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1247 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1248 vlib_node_runtime_t * node,
1249 vlib_frame_t * frame)
1251 u32 n_left_from, *from, *to_next;
1252 snat_out2in_next_t next_index;
1253 u32 pkts_processed = 0;
1254 snat_main_t *sm = &snat_main;
1255 f64 now = vlib_time_now (vm);
1256 u32 thread_index = vm->thread_index;
1257 snat_main_per_thread_data_t *per_thread_data =
1258 &sm->per_thread_data[thread_index];
1259 u32 *fragments_to_drop = 0;
1260 u32 *fragments_to_loopback = 0;
1262 from = vlib_frame_vector_args (frame);
1263 n_left_from = frame->n_vectors;
1264 next_index = node->cached_next_index;
1266 while (n_left_from > 0)
1270 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1272 while (n_left_from > 0 && n_left_to_next > 0)
1274 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1279 nat_reass_ip4_t *reass0;
1280 udp_header_t * udp0;
1281 tcp_header_t * tcp0;
1282 snat_session_key_t key0, sm0;
1283 clib_bihash_kv_8_8_t kv0, value0;
1284 snat_session_t * s0 = 0;
1285 u16 old_port0, new_port0;
1288 /* speculatively enqueue b0 to the current next frame */
1294 n_left_to_next -= 1;
1296 b0 = vlib_get_buffer (vm, bi0);
1297 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1299 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1300 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1303 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1305 next0 = SNAT_OUT2IN_NEXT_DROP;
1306 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1310 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1311 udp0 = ip4_next_header (ip0);
1312 tcp0 = (tcp_header_t *) udp0;
1313 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1315 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1320 &fragments_to_drop);
1322 if (PREDICT_FALSE (!reass0))
1324 next0 = SNAT_OUT2IN_NEXT_DROP;
1325 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1326 nat_log_notice ("maximum reassemblies exceeded");
1330 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1332 key0.addr = ip0->dst_address;
1333 key0.port = udp0->dst_port;
1334 key0.protocol = proto0;
1335 key0.fib_index = rx_fib_index0;
1336 kv0.key = key0.as_u64;
1338 if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1340 /* Try to match static mapping by external address and port,
1341 destination address and port in packet */
1342 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1345 * Send DHCP packets to the ipv4 stack, or we won't
1346 * be able to use dhcp client on the outside interface
1348 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1350 == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1352 vnet_feature_next (&next0, b0);
1356 if (!sm->forwarding_enabled)
1358 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1359 next0 = SNAT_OUT2IN_NEXT_DROP;
1364 /* Create session initiated by host from external network */
1365 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1369 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1370 next0 = SNAT_OUT2IN_NEXT_DROP;
1373 reass0->sess_index = s0 - per_thread_data->sessions;
1374 reass0->thread_index = thread_index;
1378 s0 = pool_elt_at_index (per_thread_data->sessions,
1380 reass0->sess_index = value0.value;
1382 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1386 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1388 if (nat_ip4_reass_add_fragment (reass0, bi0))
1390 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1391 nat_log_notice ("maximum fragments per reassembly exceeded");
1392 next0 = SNAT_OUT2IN_NEXT_DROP;
1398 s0 = pool_elt_at_index (per_thread_data->sessions,
1399 reass0->sess_index);
1402 old_addr0 = ip0->dst_address.as_u32;
1403 ip0->dst_address = s0->in2out.addr;
1404 new_addr0 = ip0->dst_address.as_u32;
1405 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1407 sum0 = ip0->checksum;
1408 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1410 dst_address /* changed member */);
1411 ip0->checksum = ip_csum_fold (sum0);
1413 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1415 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1417 old_port0 = tcp0->dst_port;
1418 tcp0->dst_port = s0->in2out.port;
1419 new_port0 = tcp0->dst_port;
1421 sum0 = tcp0->checksum;
1422 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1424 dst_address /* changed member */);
1426 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1427 ip4_header_t /* cheat */,
1428 length /* changed member */);
1429 tcp0->checksum = ip_csum_fold(sum0);
1433 old_port0 = udp0->dst_port;
1434 udp0->dst_port = s0->in2out.port;
1440 nat44_session_update_counters (s0, now,
1441 vlib_buffer_length_in_chain (vm, b0));
1442 /* Per-user LRU list maintenance */
1443 nat44_session_update_lru (sm, s0, thread_index);
1446 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1447 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1449 nat44_out2in_reass_trace_t *t =
1450 vlib_add_trace (vm, node, b0, sizeof (*t));
1451 t->cached = cached0;
1452 t->sw_if_index = sw_if_index0;
1453 t->next_index = next0;
1463 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1465 /* verify speculative enqueue, maybe switch current next frame */
1466 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1467 to_next, n_left_to_next,
1471 if (n_left_from == 0 && vec_len (fragments_to_loopback))
1473 from = vlib_frame_vector_args (frame);
1474 u32 len = vec_len (fragments_to_loopback);
1475 if (len <= VLIB_FRAME_SIZE)
1477 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1479 vec_reset_length (fragments_to_loopback);
1484 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1485 sizeof (u32) * VLIB_FRAME_SIZE);
1486 n_left_from = VLIB_FRAME_SIZE;
1487 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1492 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1495 vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1496 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1499 nat_send_all_to_node (vm, fragments_to_drop, node,
1500 &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1501 SNAT_OUT2IN_NEXT_DROP);
1503 vec_free (fragments_to_drop);
1504 vec_free (fragments_to_loopback);
1505 return frame->n_vectors;
1508 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1509 .function = nat44_out2in_reass_node_fn,
1510 .name = "nat44-out2in-reass",
1511 .vector_size = sizeof (u32),
1512 .format_trace = format_nat44_out2in_reass_trace,
1513 .type = VLIB_NODE_TYPE_INTERNAL,
1515 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1516 .error_strings = snat_out2in_error_strings,
1518 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1520 /* edit / add dispositions here */
1522 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1523 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1524 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1525 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1528 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1529 nat44_out2in_reass_node_fn);
1531 /*******************************/
1532 /*** endpoint-dependent mode ***/
1533 /*******************************/
1535 NAT44_ED_OUT2IN_NEXT_DROP,
1536 NAT44_ED_OUT2IN_NEXT_LOOKUP,
1537 NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
1538 NAT44_ED_OUT2IN_NEXT_IN2OUT,
1539 NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
1540 NAT44_ED_OUT2IN_N_NEXT,
1541 } nat44_ed_out2in_next_t;
1548 } nat44_ed_out2in_trace_t;
1551 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
1553 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1554 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1555 nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
1558 tag = t->is_slow_path ? "NAT44_OUT2IN_SLOW_PATH" : "NAT44_OUT2IN_FAST_PATH";
1560 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
1561 t->sw_if_index, t->next_index, t->session_index);
1566 static snat_session_t *
1567 create_session_for_static_mapping_ed (snat_main_t * sm,
1569 snat_session_key_t l_key,
1570 snat_session_key_t e_key,
1571 vlib_node_runtime_t * node,
1573 twice_nat_type_t twice_nat,
1580 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1581 clib_bihash_kv_16_8_t kv;
1582 snat_session_key_t eh_key;
1585 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1587 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1588 nat_log_notice ("maximum sessions exceeded");
1592 u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
1595 nat_log_warn ("create NAT user failed");
1599 s = nat_session_alloc_or_recycle (sm, u, thread_index);
1602 nat_log_warn ("create NAT session failed");
1606 ip = vlib_buffer_get_current (b);
1607 udp = ip4_next_header (ip);
1609 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1610 s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
1611 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1613 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1614 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1615 s->outside_address_index = ~0;
1618 s->in2out.protocol = s->out2in.protocol;
1619 user_session_increment (sm, u, 1);
1621 /* Add to lookup tables */
1622 make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
1623 e_key.fib_index, e_key.port, s->ext_host_port);
1624 kv.value = s - tsm->sessions;
1625 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 1))
1626 nat_log_notice ("out2in-ed key add failed");
1628 if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
1629 ip->src_address.as_u32 == l_key.addr.as_u32))
1631 eh_key.protocol = e_key.protocol;
1632 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1633 thread_index, &eh_key,
1635 sm->port_per_thread,
1636 tsm->snat_thread_index))
1638 b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1639 nat44_delete_session (sm, s, thread_index);
1640 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
1641 nat_log_notice ("out2in-ed key del failed");
1644 s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1645 s->ext_host_nat_port = eh_key.port;
1646 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1647 make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
1648 l_key.fib_index, l_key.port, s->ext_host_nat_port);
1652 make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
1653 l_key.fib_index, l_key.port, s->ext_host_port);
1655 kv.value = s - tsm->sessions;
1656 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1657 nat_log_notice ("in2out-ed key add failed");
1662 static_always_inline int
1663 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
1665 icmp46_header_t *icmp0;
1666 nat_ed_ses_key_t key0;
1667 icmp_echo_header_t *echo0, *inner_echo0 = 0;
1668 ip4_header_t *inner_ip0;
1669 void *l4_header = 0;
1670 icmp46_header_t *inner_icmp0;
1672 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1673 echo0 = (icmp_echo_header_t *)(icmp0+1);
1675 if (!icmp_is_error_message (icmp0))
1677 key0.proto = IP_PROTOCOL_ICMP;
1678 key0.l_addr = ip0->dst_address;
1679 key0.r_addr = ip0->src_address;
1680 key0.l_port = echo0->identifier;
1685 inner_ip0 = (ip4_header_t *)(echo0+1);
1686 l4_header = ip4_next_header (inner_ip0);
1687 key0.proto = inner_ip0->protocol;
1688 key0.l_addr = inner_ip0->src_address;
1689 key0.r_addr = inner_ip0->dst_address;
1690 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
1692 case SNAT_PROTOCOL_ICMP:
1693 inner_icmp0 = (icmp46_header_t*)l4_header;
1694 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1695 key0.l_port = inner_echo0->identifier;
1698 case SNAT_PROTOCOL_UDP:
1699 case SNAT_PROTOCOL_TCP:
1700 key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
1701 key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1712 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
1713 u16 dst_port, u32 thread_index)
1715 clib_bihash_kv_16_8_t kv, value;
1716 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1718 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
1719 sm->inside_fib_index, src_port, dst_port);
1720 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1727 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
1730 nat_ed_ses_key_t key;
1731 clib_bihash_kv_16_8_t kv, value;
1734 snat_session_t *s = 0;
1735 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1736 f64 now = vlib_time_now (sm->vlib_main);
1738 if (ip->protocol == IP_PROTOCOL_ICMP)
1740 if (icmp_get_ed_key (ip, &key))
1743 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
1745 udp = ip4_next_header(ip);
1746 key.r_addr = ip->src_address;
1747 key.l_addr = ip->dst_address;
1748 key.proto = ip->protocol;
1749 key.l_port = udp->dst_port;
1750 key.r_port = udp->src_port;
1754 key.r_addr = ip->src_address;
1755 key.l_addr = ip->dst_address;
1756 key.proto = ip->protocol;
1757 key.l_port = key.r_port = 0;
1760 kv.key[0] = key.as_u64[0];
1761 kv.key[1] = key.as_u64[1];
1763 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1765 s = pool_elt_at_index (tsm->sessions, value.value);
1769 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1772 u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
1776 nat_log_warn ("create NAT user failed");
1780 s = nat_session_alloc_or_recycle (sm, u, thread_index);
1783 nat_log_warn ("create NAT session failed");
1787 s->ext_host_addr = key.r_addr;
1788 s->ext_host_port = key.r_port;
1789 s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
1790 s->outside_address_index = ~0;
1791 s->out2in.addr = key.l_addr;
1792 s->out2in.port = key.l_port;
1793 s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
1794 s->out2in.fib_index = 0;
1795 s->in2out = s->out2in;
1796 user_session_increment (sm, u, 0);
1798 kv.value = s - tsm->sessions;
1799 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1800 nat_log_notice ("in2out_ed key add failed");
1803 if (ip->protocol == IP_PROTOCOL_TCP)
1805 tcp_header_t *tcp = ip4_next_header(ip);
1806 if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
1810 /* Per-user LRU list maintenance */
1811 nat44_session_update_lru (sm, s, thread_index);
1813 nat44_session_update_counters (s, now, 0);
1817 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
1818 u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
1819 u8 * p_proto, snat_session_key_t * p_value,
1820 u8 * p_dont_translate, void * d, void * e)
1822 u32 next = ~0, sw_if_index, rx_fib_index;
1823 icmp46_header_t *icmp;
1824 nat_ed_ses_key_t key;
1825 clib_bihash_kv_16_8_t kv, value;
1826 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1827 snat_session_t *s = 0;
1828 u8 dont_translate = 0, is_addr_only;
1829 snat_session_key_t e_key, l_key;
1831 icmp = (icmp46_header_t *) ip4_next_header (ip);
1832 sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
1833 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
1835 if (icmp_get_ed_key (ip, &key))
1837 b->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1838 next = SNAT_OUT2IN_NEXT_DROP;
1841 key.fib_index = rx_fib_index;
1842 kv.key[0] = key.as_u64[0];
1843 kv.key[1] = key.as_u64[1];
1845 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
1847 /* Try to match static mapping */
1848 e_key.addr = ip->dst_address;
1849 e_key.port = key.l_port;
1850 e_key.protocol = ip_proto_to_snat_proto (key.proto);
1851 e_key.fib_index = rx_fib_index;
1852 if (snat_static_mapping_match(sm, e_key, &l_key, 1, &is_addr_only, 0, 0))
1854 if (!sm->forwarding_enabled)
1856 /* Don't NAT packet aimed at the intfc address */
1857 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index,
1858 ip->dst_address.as_u32)))
1863 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1864 next = NAT44_ED_OUT2IN_NEXT_DROP;
1870 if (next_src_nat(sm, ip, key.proto, key.l_port, key.r_port, thread_index))
1872 next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
1875 create_bypass_for_fwd(sm, ip, rx_fib_index, thread_index);
1880 if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1881 (icmp->type != ICMP4_echo_request || !is_addr_only)))
1883 b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1884 next = NAT44_ED_OUT2IN_NEXT_DROP;
1888 /* Create session initiated by host from external network */
1889 s = create_session_for_static_mapping_ed(sm, b, l_key, e_key, node,
1890 thread_index, 0, 0);
1894 next = NAT44_ED_OUT2IN_NEXT_DROP;
1900 if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
1901 icmp->type != ICMP4_echo_request &&
1902 !icmp_is_error_message (icmp)))
1904 b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
1905 next = SNAT_OUT2IN_NEXT_DROP;
1909 s = pool_elt_at_index (tsm->sessions, value.value);
1912 *p_proto = ip_proto_to_snat_proto (key.proto);
1915 *p_value = s->in2out;
1916 *p_dont_translate = dont_translate;
1918 *(snat_session_t**)d = s;
1922 static snat_session_t *
1923 nat44_ed_out2in_unknown_proto (snat_main_t *sm,
1930 vlib_node_runtime_t * node)
1932 clib_bihash_kv_8_8_t kv, value;
1933 clib_bihash_kv_16_8_t s_kv, s_value;
1934 snat_static_mapping_t *m;
1935 u32 old_addr, new_addr;
1938 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1941 old_addr = ip->dst_address.as_u32;
1943 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
1944 rx_fib_index, 0, 0);
1946 if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
1948 s = pool_elt_at_index (tsm->sessions, s_value.value);
1949 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1953 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1955 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1956 nat_log_notice ("maximum sessions exceeded");
1960 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1961 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1963 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1967 m = pool_elt_at_index (sm->static_mappings, value.value);
1969 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1971 u = nat_user_get_or_create (sm, &ip->src_address, m->fib_index,
1975 nat_log_warn ("create NAT user failed");
1979 /* Create a new session */
1980 s = nat_session_alloc_or_recycle (sm, u, thread_index);
1983 nat_log_warn ("create NAT session failed");
1987 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1988 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1989 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1990 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1991 s->outside_address_index = ~0;
1992 s->out2in.addr.as_u32 = old_addr;
1993 s->out2in.fib_index = rx_fib_index;
1994 s->in2out.addr.as_u32 = new_addr;
1995 s->in2out.fib_index = m->fib_index;
1996 s->in2out.port = s->out2in.port = ip->protocol;
1997 user_session_increment (sm, u, 1);
1999 /* Add to lookup tables */
2000 s_kv.value = s - tsm->sessions;
2001 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2002 nat_log_notice ("out2in key add failed");
2004 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2005 m->fib_index, 0, 0);
2006 s_kv.value = s - tsm->sessions;
2007 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2008 nat_log_notice ("in2out key add failed");
2011 /* Update IP checksum */
2013 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2014 ip->checksum = ip_csum_fold (sum);
2016 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2019 nat44_session_update_counters (s, now,
2020 vlib_buffer_length_in_chain (vm, b));
2021 /* Per-user LRU list maintenance */
2022 nat44_session_update_lru (sm, s, thread_index);
2028 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
2029 vlib_node_runtime_t * node,
2030 vlib_frame_t * frame, int is_slow_path)
2032 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2033 nat44_ed_out2in_next_t next_index;
2034 snat_main_t *sm = &snat_main;
2035 f64 now = vlib_time_now (vm);
2036 u32 thread_index = vm->thread_index;
2037 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2039 stats_node_index = is_slow_path ? nat44_ed_out2in_slowpath_node.index :
2040 nat44_ed_out2in_node.index;
2042 from = vlib_frame_vector_args (frame);
2043 n_left_from = frame->n_vectors;
2044 next_index = node->cached_next_index;
2046 while (n_left_from > 0)
2050 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2052 while (n_left_from >= 4 && n_left_to_next >= 2)
2055 vlib_buffer_t *b0, *b1;
2056 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2057 u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1, new_addr1;
2058 u16 old_port0, new_port0, old_port1, new_port1;
2059 ip4_header_t *ip0, *ip1;
2060 udp_header_t *udp0, *udp1;
2061 tcp_header_t *tcp0, *tcp1;
2062 icmp46_header_t *icmp0, *icmp1;
2063 snat_session_t *s0 = 0, *s1 = 0;
2064 clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
2065 ip_csum_t sum0, sum1;
2066 snat_session_key_t e_key0, l_key0, e_key1, l_key1;
2068 twice_nat_type_t twice_nat0, twice_nat1;
2070 /* Prefetch next iteration. */
2072 vlib_buffer_t * p2, * p3;
2074 p2 = vlib_get_buffer (vm, from[2]);
2075 p3 = vlib_get_buffer (vm, from[3]);
2077 vlib_prefetch_buffer_header (p2, LOAD);
2078 vlib_prefetch_buffer_header (p3, LOAD);
2080 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2081 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2084 /* speculatively enqueue b0 and b1 to the current next frame */
2085 to_next[0] = bi0 = from[0];
2086 to_next[1] = bi1 = from[1];
2090 n_left_to_next -= 2;
2092 b0 = vlib_get_buffer (vm, bi0);
2093 b1 = vlib_get_buffer (vm, bi1);
2095 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2096 vnet_buffer (b0)->snat.flags = 0;
2097 ip0 = vlib_buffer_get_current (b0);
2099 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2100 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2103 if (PREDICT_FALSE(ip0->ttl == 1))
2105 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2106 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2107 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2109 next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2113 udp0 = ip4_next_header (ip0);
2114 tcp0 = (tcp_header_t *) udp0;
2115 icmp0 = (icmp46_header_t *) udp0;
2116 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2120 if (PREDICT_FALSE (proto0 == ~0))
2122 s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2123 thread_index, now, vm, node);
2124 if (!sm->forwarding_enabled)
2127 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2132 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2134 next0 = icmp_out2in_slow_path
2135 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2136 next0, now, thread_index, &s0);
2142 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2144 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2148 if (ip4_is_fragment (ip0))
2150 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2151 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2156 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2157 rx_fib_index0, udp0->dst_port, udp0->src_port);
2159 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2163 /* Try to match static mapping by external address and port,
2164 destination address and port in packet */
2165 e_key0.addr = ip0->dst_address;
2166 e_key0.port = udp0->dst_port;
2167 e_key0.protocol = proto0;
2168 e_key0.fib_index = rx_fib_index0;
2169 if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2170 &twice_nat0, &is_lb0))
2173 * Send DHCP packets to the ipv4 stack, or we won't
2174 * be able to use dhcp client on the outside interface
2176 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2177 && (udp0->dst_port ==
2178 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2180 vnet_feature_next (&next0, b0);
2184 if (!sm->forwarding_enabled)
2186 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2187 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2191 if (next_src_nat(sm, ip0, ip0->protocol,
2192 udp0->src_port, udp0->dst_port,
2195 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2198 create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2204 /* Create session initiated by host from external network */
2205 s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2208 twice_nat0, is_lb0);
2212 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2218 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2224 s0 = pool_elt_at_index (tsm->sessions, value0.value);
2227 old_addr0 = ip0->dst_address.as_u32;
2228 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2229 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2231 sum0 = ip0->checksum;
2232 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2234 if (PREDICT_FALSE (is_twice_nat_session (s0)))
2235 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2236 s0->ext_host_nat_addr.as_u32, ip4_header_t,
2238 ip0->checksum = ip_csum_fold (sum0);
2240 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2242 old_port0 = tcp0->dst_port;
2243 new_port0 = tcp0->dst_port = s0->in2out.port;
2245 sum0 = tcp0->checksum;
2246 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2248 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2250 if (is_twice_nat_session (s0))
2252 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2253 s0->ext_host_nat_addr.as_u32,
2254 ip4_header_t, dst_address);
2255 sum0 = ip_csum_update (sum0, tcp0->src_port,
2256 s0->ext_host_nat_port, ip4_header_t,
2258 tcp0->src_port = s0->ext_host_nat_port;
2259 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2261 tcp0->checksum = ip_csum_fold(sum0);
2262 if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2267 udp0->dst_port = s0->in2out.port;
2268 if (is_twice_nat_session (s0))
2270 udp0->src_port = s0->ext_host_nat_port;
2271 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2277 nat44_session_update_counters (s0, now,
2278 vlib_buffer_length_in_chain (vm, b0));
2279 /* Per-user LRU list maintenance */
2280 nat44_session_update_lru (sm, s0, thread_index);
2283 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2284 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2286 nat44_ed_out2in_trace_t *t =
2287 vlib_add_trace (vm, node, b0, sizeof (*t));
2288 t->is_slow_path = is_slow_path;
2289 t->sw_if_index = sw_if_index0;
2290 t->next_index = next0;
2291 t->session_index = ~0;
2293 t->session_index = s0 - tsm->sessions;
2296 pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2298 next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2299 vnet_buffer (b1)->snat.flags = 0;
2300 ip1 = vlib_buffer_get_current (b1);
2302 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2303 rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2306 if (PREDICT_FALSE(ip1->ttl == 1))
2308 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2309 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2310 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2312 next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2316 udp1 = ip4_next_header (ip1);
2317 tcp1 = (tcp_header_t *) udp1;
2318 icmp1 = (icmp46_header_t *) udp1;
2319 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2323 if (PREDICT_FALSE (proto1 == ~0))
2325 s1 = nat44_ed_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
2326 thread_index, now, vm, node);
2327 if (!sm->forwarding_enabled)
2330 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2335 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
2337 next1 = icmp_out2in_slow_path
2338 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
2339 next1, now, thread_index, &s1);
2345 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
2347 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2351 if (ip4_is_fragment (ip1))
2353 b1->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2354 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2359 make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address, ip1->protocol,
2360 rx_fib_index1, udp1->dst_port, udp1->src_port);
2362 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
2366 /* Try to match static mapping by external address and port,
2367 destination address and port in packet */
2368 e_key1.addr = ip1->dst_address;
2369 e_key1.port = udp1->dst_port;
2370 e_key1.protocol = proto1;
2371 e_key1.fib_index = rx_fib_index1;
2372 if (snat_static_mapping_match(sm, e_key1, &l_key1, 1, 0,
2373 &twice_nat1, &is_lb1))
2376 * Send DHCP packets to the ipv4 stack, or we won't
2377 * be able to use dhcp client on the outside interface
2379 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
2380 && (udp1->dst_port ==
2381 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2383 vnet_feature_next (&next1, b1);
2387 if (!sm->forwarding_enabled)
2389 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2390 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2394 if (next_src_nat(sm, ip1, ip1->protocol,
2395 udp1->src_port, udp1->dst_port,
2398 next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2401 create_bypass_for_fwd(sm, ip1, rx_fib_index1,
2407 /* Create session initiated by host from external network */
2408 s1 = create_session_for_static_mapping_ed(sm, b1, l_key1,
2411 twice_nat1, is_lb1);
2415 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2421 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2427 s1 = pool_elt_at_index (tsm->sessions, value1.value);
2430 old_addr1 = ip1->dst_address.as_u32;
2431 new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
2432 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
2434 sum1 = ip1->checksum;
2435 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2437 if (PREDICT_FALSE (is_twice_nat_session (s1)))
2438 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2439 s1->ext_host_nat_addr.as_u32, ip4_header_t,
2441 ip1->checksum = ip_csum_fold (sum1);
2443 if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
2445 old_port1 = tcp1->dst_port;
2446 new_port1 = tcp1->dst_port = s1->in2out.port;
2448 sum1 = tcp1->checksum;
2449 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2451 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
2453 if (is_twice_nat_session (s1))
2455 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2456 s1->ext_host_nat_addr.as_u32,
2457 ip4_header_t, dst_address);
2458 sum1 = ip_csum_update (sum1, tcp1->src_port,
2459 s1->ext_host_nat_port, ip4_header_t,
2461 tcp1->src_port = s1->ext_host_nat_port;
2462 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2464 tcp1->checksum = ip_csum_fold(sum1);
2465 if (nat44_set_tcp_session_state_o2i (sm, s1, tcp1, thread_index))
2470 udp1->dst_port = s1->in2out.port;
2471 if (is_twice_nat_session (s1))
2473 udp1->src_port = s1->ext_host_nat_port;
2474 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2480 nat44_session_update_counters (s1, now,
2481 vlib_buffer_length_in_chain (vm, b1));
2482 /* Per-user LRU list maintenance */
2483 nat44_session_update_lru (sm, s1, thread_index);
2486 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2487 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2489 nat44_ed_out2in_trace_t *t =
2490 vlib_add_trace (vm, node, b1, sizeof (*t));
2491 t->is_slow_path = is_slow_path;
2492 t->sw_if_index = sw_if_index1;
2493 t->next_index = next1;
2494 t->session_index = ~0;
2496 t->session_index = s1 - tsm->sessions;
2499 pkts_processed += next1 != NAT44_ED_OUT2IN_NEXT_DROP;
2501 /* verify speculative enqueues, maybe switch current next frame */
2502 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2503 to_next, n_left_to_next,
2504 bi0, bi1, next0, next1);
2507 while (n_left_from > 0 && n_left_to_next > 0)
2511 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2512 u16 old_port0, new_port0;
2516 icmp46_header_t * icmp0;
2517 snat_session_t *s0 = 0;
2518 clib_bihash_kv_16_8_t kv0, value0;
2520 snat_session_key_t e_key0, l_key0;
2522 twice_nat_type_t twice_nat0;
2524 /* speculatively enqueue b0 to the current next frame */
2530 n_left_to_next -= 1;
2532 b0 = vlib_get_buffer (vm, bi0);
2533 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2534 vnet_buffer (b0)->snat.flags = 0;
2535 ip0 = vlib_buffer_get_current (b0);
2537 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2538 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2541 if (PREDICT_FALSE(ip0->ttl == 1))
2543 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2544 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2545 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2547 next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2551 udp0 = ip4_next_header (ip0);
2552 tcp0 = (tcp_header_t *) udp0;
2553 icmp0 = (icmp46_header_t *) udp0;
2554 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2558 if (PREDICT_FALSE (proto0 == ~0))
2560 s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2561 thread_index, now, vm, node);
2562 if (!sm->forwarding_enabled)
2565 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2570 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2572 next0 = icmp_out2in_slow_path
2573 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2574 next0, now, thread_index, &s0);
2580 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2582 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2586 if (ip4_is_fragment (ip0))
2588 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2589 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2594 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2595 rx_fib_index0, udp0->dst_port, udp0->src_port);
2597 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2601 /* Try to match static mapping by external address and port,
2602 destination address and port in packet */
2603 e_key0.addr = ip0->dst_address;
2604 e_key0.port = udp0->dst_port;
2605 e_key0.protocol = proto0;
2606 e_key0.fib_index = rx_fib_index0;
2607 if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2608 &twice_nat0, &is_lb0))
2611 * Send DHCP packets to the ipv4 stack, or we won't
2612 * be able to use dhcp client on the outside interface
2614 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2615 && (udp0->dst_port ==
2616 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2618 vnet_feature_next (&next0, b0);
2622 if (!sm->forwarding_enabled)
2624 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2625 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2629 if (next_src_nat(sm, ip0, ip0->protocol,
2630 udp0->src_port, udp0->dst_port,
2633 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2636 create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2642 /* Create session initiated by host from external network */
2643 s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2646 twice_nat0, is_lb0);
2650 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2656 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2662 s0 = pool_elt_at_index (tsm->sessions, value0.value);
2665 old_addr0 = ip0->dst_address.as_u32;
2666 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2667 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2669 sum0 = ip0->checksum;
2670 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2672 if (PREDICT_FALSE (is_twice_nat_session (s0)))
2673 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2674 s0->ext_host_nat_addr.as_u32, ip4_header_t,
2676 ip0->checksum = ip_csum_fold (sum0);
2678 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2680 old_port0 = tcp0->dst_port;
2681 new_port0 = tcp0->dst_port = s0->in2out.port;
2683 sum0 = tcp0->checksum;
2684 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2686 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2688 if (is_twice_nat_session (s0))
2690 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2691 s0->ext_host_nat_addr.as_u32,
2692 ip4_header_t, dst_address);
2693 sum0 = ip_csum_update (sum0, tcp0->src_port,
2694 s0->ext_host_nat_port, ip4_header_t,
2696 tcp0->src_port = s0->ext_host_nat_port;
2697 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2699 tcp0->checksum = ip_csum_fold(sum0);
2700 if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2705 udp0->dst_port = s0->in2out.port;
2706 if (is_twice_nat_session (s0))
2708 udp0->src_port = s0->ext_host_nat_port;
2709 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2715 nat44_session_update_counters (s0, now,
2716 vlib_buffer_length_in_chain (vm, b0));
2717 /* Per-user LRU list maintenance */
2718 nat44_session_update_lru (sm, s0, thread_index);
2721 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2722 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2724 nat44_ed_out2in_trace_t *t =
2725 vlib_add_trace (vm, node, b0, sizeof (*t));
2726 t->is_slow_path = is_slow_path;
2727 t->sw_if_index = sw_if_index0;
2728 t->next_index = next0;
2729 t->session_index = ~0;
2731 t->session_index = s0 - tsm->sessions;
2734 pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2736 /* verify speculative enqueue, maybe switch current next frame */
2737 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2738 to_next, n_left_to_next,
2742 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2745 vlib_node_increment_counter (vm, stats_node_index,
2746 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2748 return frame->n_vectors;
2752 nat44_ed_out2in_fast_path_fn (vlib_main_t * vm,
2753 vlib_node_runtime_t * node,
2754 vlib_frame_t * frame)
2756 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
2759 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
2760 .function = nat44_ed_out2in_fast_path_fn,
2761 .name = "nat44-ed-out2in",
2762 .vector_size = sizeof (u32),
2763 .format_trace = format_nat44_ed_out2in_trace,
2764 .type = VLIB_NODE_TYPE_INTERNAL,
2766 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2767 .error_strings = snat_out2in_error_strings,
2769 .runtime_data_bytes = sizeof (snat_runtime_t),
2771 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2773 /* edit / add dispositions here */
2775 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2776 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2777 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2778 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2779 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2783 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_node, nat44_ed_out2in_fast_path_fn);
2786 nat44_ed_out2in_slow_path_fn (vlib_main_t * vm,
2787 vlib_node_runtime_t * node,
2788 vlib_frame_t * frame)
2790 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
2793 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
2794 .function = nat44_ed_out2in_slow_path_fn,
2795 .name = "nat44-ed-out2in-slowpath",
2796 .vector_size = sizeof (u32),
2797 .format_trace = format_nat44_ed_out2in_trace,
2798 .type = VLIB_NODE_TYPE_INTERNAL,
2800 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2801 .error_strings = snat_out2in_error_strings,
2803 .runtime_data_bytes = sizeof (snat_runtime_t),
2805 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2807 /* edit / add dispositions here */
2809 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2810 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2811 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2812 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2813 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2817 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_slowpath_node,
2818 nat44_ed_out2in_slow_path_fn);
2820 /**************************/
2821 /*** deterministic mode ***/
2822 /**************************/
2824 snat_det_out2in_node_fn (vlib_main_t * vm,
2825 vlib_node_runtime_t * node,
2826 vlib_frame_t * frame)
2828 u32 n_left_from, * from, * to_next;
2829 snat_out2in_next_t next_index;
2830 u32 pkts_processed = 0;
2831 snat_main_t * sm = &snat_main;
2832 u32 thread_index = vm->thread_index;
2834 from = vlib_frame_vector_args (frame);
2835 n_left_from = frame->n_vectors;
2836 next_index = node->cached_next_index;
2838 while (n_left_from > 0)
2842 vlib_get_next_frame (vm, node, next_index,
2843 to_next, n_left_to_next);
2845 while (n_left_from >= 4 && n_left_to_next >= 2)
2848 vlib_buffer_t * b0, * b1;
2849 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
2850 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
2851 u32 sw_if_index0, sw_if_index1;
2852 ip4_header_t * ip0, * ip1;
2853 ip_csum_t sum0, sum1;
2854 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2855 u16 new_port0, old_port0, old_port1, new_port1;
2856 udp_header_t * udp0, * udp1;
2857 tcp_header_t * tcp0, * tcp1;
2859 snat_det_out_key_t key0, key1;
2860 snat_det_map_t * dm0, * dm1;
2861 snat_det_session_t * ses0 = 0, * ses1 = 0;
2862 u32 rx_fib_index0, rx_fib_index1;
2863 icmp46_header_t * icmp0, * icmp1;
2865 /* Prefetch next iteration. */
2867 vlib_buffer_t * p2, * p3;
2869 p2 = vlib_get_buffer (vm, from[2]);
2870 p3 = vlib_get_buffer (vm, from[3]);
2872 vlib_prefetch_buffer_header (p2, LOAD);
2873 vlib_prefetch_buffer_header (p3, LOAD);
2875 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2876 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2879 /* speculatively enqueue b0 and b1 to the current next frame */
2880 to_next[0] = bi0 = from[0];
2881 to_next[1] = bi1 = from[1];
2885 n_left_to_next -= 2;
2887 b0 = vlib_get_buffer (vm, bi0);
2888 b1 = vlib_get_buffer (vm, bi1);
2890 ip0 = vlib_buffer_get_current (b0);
2891 udp0 = ip4_next_header (ip0);
2892 tcp0 = (tcp_header_t *) udp0;
2894 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2896 if (PREDICT_FALSE(ip0->ttl == 1))
2898 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2899 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2900 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2902 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
2906 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2908 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2910 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2911 icmp0 = (icmp46_header_t *) udp0;
2913 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
2914 rx_fib_index0, node, next0, thread_index,
2919 key0.ext_host_addr = ip0->src_address;
2920 key0.ext_host_port = tcp0->src;
2921 key0.out_port = tcp0->dst;
2923 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
2924 if (PREDICT_FALSE(!dm0))
2926 nat_log_info ("unknown dst address: %U",
2927 format_ip4_address, &ip0->dst_address);
2928 next0 = SNAT_OUT2IN_NEXT_DROP;
2929 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2933 snat_det_reverse(dm0, &ip0->dst_address,
2934 clib_net_to_host_u16(tcp0->dst), &new_addr0);
2936 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
2937 if (PREDICT_FALSE(!ses0))
2939 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
2940 format_ip4_address, &ip0->src_address,
2941 clib_net_to_host_u16 (tcp0->src),
2942 format_ip4_address, &ip0->dst_address,
2943 clib_net_to_host_u16 (tcp0->dst),
2944 format_ip4_address, &new_addr0);
2945 next0 = SNAT_OUT2IN_NEXT_DROP;
2946 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2949 new_port0 = ses0->in_port;
2951 old_addr0 = ip0->dst_address;
2952 ip0->dst_address = new_addr0;
2953 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
2955 sum0 = ip0->checksum;
2956 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2958 dst_address /* changed member */);
2959 ip0->checksum = ip_csum_fold (sum0);
2961 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2963 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2964 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
2965 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
2966 snat_det_ses_close(dm0, ses0);
2968 old_port0 = tcp0->dst;
2969 tcp0->dst = new_port0;
2971 sum0 = tcp0->checksum;
2972 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2974 dst_address /* changed member */);
2976 sum0 = ip_csum_update (sum0, old_port0, new_port0,
2977 ip4_header_t /* cheat */,
2978 length /* changed member */);
2979 tcp0->checksum = ip_csum_fold(sum0);
2983 old_port0 = udp0->dst_port;
2984 udp0->dst_port = new_port0;
2990 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2991 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2993 snat_out2in_trace_t *t =
2994 vlib_add_trace (vm, node, b0, sizeof (*t));
2995 t->sw_if_index = sw_if_index0;
2996 t->next_index = next0;
2997 t->session_index = ~0;
2999 t->session_index = ses0 - dm0->sessions;
3002 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3004 b1 = vlib_get_buffer (vm, bi1);
3006 ip1 = vlib_buffer_get_current (b1);
3007 udp1 = ip4_next_header (ip1);
3008 tcp1 = (tcp_header_t *) udp1;
3010 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3012 if (PREDICT_FALSE(ip1->ttl == 1))
3014 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3015 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3016 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3018 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3022 proto1 = ip_proto_to_snat_proto (ip1->protocol);
3024 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3026 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3027 icmp1 = (icmp46_header_t *) udp1;
3029 next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
3030 rx_fib_index1, node, next1, thread_index,
3035 key1.ext_host_addr = ip1->src_address;
3036 key1.ext_host_port = tcp1->src;
3037 key1.out_port = tcp1->dst;
3039 dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
3040 if (PREDICT_FALSE(!dm1))
3042 nat_log_info ("unknown dst address: %U",
3043 format_ip4_address, &ip1->dst_address);
3044 next1 = SNAT_OUT2IN_NEXT_DROP;
3045 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3049 snat_det_reverse(dm1, &ip1->dst_address,
3050 clib_net_to_host_u16(tcp1->dst), &new_addr1);
3052 ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
3053 if (PREDICT_FALSE(!ses1))
3055 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3056 format_ip4_address, &ip1->src_address,
3057 clib_net_to_host_u16 (tcp1->src),
3058 format_ip4_address, &ip1->dst_address,
3059 clib_net_to_host_u16 (tcp1->dst),
3060 format_ip4_address, &new_addr1);
3061 next1 = SNAT_OUT2IN_NEXT_DROP;
3062 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3065 new_port1 = ses1->in_port;
3067 old_addr1 = ip1->dst_address;
3068 ip1->dst_address = new_addr1;
3069 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3071 sum1 = ip1->checksum;
3072 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3074 dst_address /* changed member */);
3075 ip1->checksum = ip_csum_fold (sum1);
3077 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3079 if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3080 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3081 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
3082 snat_det_ses_close(dm1, ses1);
3084 old_port1 = tcp1->dst;
3085 tcp1->dst = new_port1;
3087 sum1 = tcp1->checksum;
3088 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3090 dst_address /* changed member */);
3092 sum1 = ip_csum_update (sum1, old_port1, new_port1,
3093 ip4_header_t /* cheat */,
3094 length /* changed member */);
3095 tcp1->checksum = ip_csum_fold(sum1);
3099 old_port1 = udp1->dst_port;
3100 udp1->dst_port = new_port1;
3106 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3107 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3109 snat_out2in_trace_t *t =
3110 vlib_add_trace (vm, node, b1, sizeof (*t));
3111 t->sw_if_index = sw_if_index1;
3112 t->next_index = next1;
3113 t->session_index = ~0;
3115 t->session_index = ses1 - dm1->sessions;
3118 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
3120 /* verify speculative enqueues, maybe switch current next frame */
3121 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3122 to_next, n_left_to_next,
3123 bi0, bi1, next0, next1);
3126 while (n_left_from > 0 && n_left_to_next > 0)
3130 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
3134 ip4_address_t new_addr0, old_addr0;
3135 u16 new_port0, old_port0;
3136 udp_header_t * udp0;
3137 tcp_header_t * tcp0;
3139 snat_det_out_key_t key0;
3140 snat_det_map_t * dm0;
3141 snat_det_session_t * ses0 = 0;
3143 icmp46_header_t * icmp0;
3145 /* speculatively enqueue b0 to the current next frame */
3151 n_left_to_next -= 1;
3153 b0 = vlib_get_buffer (vm, bi0);
3155 ip0 = vlib_buffer_get_current (b0);
3156 udp0 = ip4_next_header (ip0);
3157 tcp0 = (tcp_header_t *) udp0;
3159 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3161 if (PREDICT_FALSE(ip0->ttl == 1))
3163 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3164 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3165 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3167 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3171 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3173 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3175 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3176 icmp0 = (icmp46_header_t *) udp0;
3178 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3179 rx_fib_index0, node, next0, thread_index,
3184 key0.ext_host_addr = ip0->src_address;
3185 key0.ext_host_port = tcp0->src;
3186 key0.out_port = tcp0->dst;
3188 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
3189 if (PREDICT_FALSE(!dm0))
3191 nat_log_info ("unknown dst address: %U",
3192 format_ip4_address, &ip0->dst_address);
3193 next0 = SNAT_OUT2IN_NEXT_DROP;
3194 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3198 snat_det_reverse(dm0, &ip0->dst_address,
3199 clib_net_to_host_u16(tcp0->dst), &new_addr0);
3201 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3202 if (PREDICT_FALSE(!ses0))
3204 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3205 format_ip4_address, &ip0->src_address,
3206 clib_net_to_host_u16 (tcp0->src),
3207 format_ip4_address, &ip0->dst_address,
3208 clib_net_to_host_u16 (tcp0->dst),
3209 format_ip4_address, &new_addr0);
3210 next0 = SNAT_OUT2IN_NEXT_DROP;
3211 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3214 new_port0 = ses0->in_port;
3216 old_addr0 = ip0->dst_address;
3217 ip0->dst_address = new_addr0;
3218 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3220 sum0 = ip0->checksum;
3221 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3223 dst_address /* changed member */);
3224 ip0->checksum = ip_csum_fold (sum0);
3226 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3228 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3229 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3230 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
3231 snat_det_ses_close(dm0, ses0);
3233 old_port0 = tcp0->dst;
3234 tcp0->dst = new_port0;
3236 sum0 = tcp0->checksum;
3237 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3239 dst_address /* changed member */);
3241 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3242 ip4_header_t /* cheat */,
3243 length /* changed member */);
3244 tcp0->checksum = ip_csum_fold(sum0);
3248 old_port0 = udp0->dst_port;
3249 udp0->dst_port = new_port0;
3255 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3256 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3258 snat_out2in_trace_t *t =
3259 vlib_add_trace (vm, node, b0, sizeof (*t));
3260 t->sw_if_index = sw_if_index0;
3261 t->next_index = next0;
3262 t->session_index = ~0;
3264 t->session_index = ses0 - dm0->sessions;
3267 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3269 /* verify speculative enqueue, maybe switch current next frame */
3270 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3271 to_next, n_left_to_next,
3275 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3278 vlib_node_increment_counter (vm, snat_det_out2in_node.index,
3279 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3281 return frame->n_vectors;
3284 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
3285 .function = snat_det_out2in_node_fn,
3286 .name = "nat44-det-out2in",
3287 .vector_size = sizeof (u32),
3288 .format_trace = format_snat_out2in_trace,
3289 .type = VLIB_NODE_TYPE_INTERNAL,
3291 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3292 .error_strings = snat_out2in_error_strings,
3294 .runtime_data_bytes = sizeof (snat_runtime_t),
3296 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3298 /* edit / add dispositions here */
3300 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3301 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3302 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3303 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3306 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
3309 * Get address and port values to be used for ICMP packet translation
3310 * and create session if needed
3312 * @param[in,out] sm NAT main
3313 * @param[in,out] node NAT node runtime
3314 * @param[in] thread_index thread index
3315 * @param[in,out] b0 buffer containing packet to be translated
3316 * @param[out] p_proto protocol used for matching
3317 * @param[out] p_value address and port after NAT translation
3318 * @param[out] p_dont_translate if packet should not be translated
3319 * @param d optional parameter
3320 * @param e optional parameter
3322 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
3323 u32 thread_index, vlib_buffer_t *b0,
3324 ip4_header_t *ip0, u8 *p_proto,
3325 snat_session_key_t *p_value,
3326 u8 *p_dont_translate, void *d, void *e)
3328 icmp46_header_t *icmp0;
3331 snat_det_out_key_t key0;
3332 u8 dont_translate = 0;
3334 icmp_echo_header_t *echo0, *inner_echo0 = 0;
3335 ip4_header_t *inner_ip0;
3336 void *l4_header = 0;
3337 icmp46_header_t *inner_icmp0;
3338 snat_det_map_t * dm0 = 0;
3339 ip4_address_t new_addr0 = {{0}};
3340 snat_det_session_t * ses0 = 0;
3341 ip4_address_t out_addr;
3343 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3344 echo0 = (icmp_echo_header_t *)(icmp0+1);
3345 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3347 if (!icmp_is_error_message (icmp0))
3349 protocol = SNAT_PROTOCOL_ICMP;
3350 key0.ext_host_addr = ip0->src_address;
3351 key0.ext_host_port = 0;
3352 key0.out_port = echo0->identifier;
3353 out_addr = ip0->dst_address;
3357 inner_ip0 = (ip4_header_t *)(echo0+1);
3358 l4_header = ip4_next_header (inner_ip0);
3359 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3360 key0.ext_host_addr = inner_ip0->dst_address;
3361 out_addr = inner_ip0->src_address;
3364 case SNAT_PROTOCOL_ICMP:
3365 inner_icmp0 = (icmp46_header_t*)l4_header;
3366 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3367 key0.ext_host_port = 0;
3368 key0.out_port = inner_echo0->identifier;
3370 case SNAT_PROTOCOL_UDP:
3371 case SNAT_PROTOCOL_TCP:
3372 key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3373 key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
3376 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
3377 next0 = SNAT_OUT2IN_NEXT_DROP;
3382 dm0 = snat_det_map_by_out(sm, &out_addr);
3383 if (PREDICT_FALSE(!dm0))
3385 /* Don't NAT packet aimed at the intfc address */
3386 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3387 ip0->dst_address.as_u32)))
3392 nat_log_info ("unknown dst address: %U",
3393 format_ip4_address, &ip0->dst_address);
3397 snat_det_reverse(dm0, &ip0->dst_address,
3398 clib_net_to_host_u16(key0.out_port), &new_addr0);
3400 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3401 if (PREDICT_FALSE(!ses0))
3403 /* Don't NAT packet aimed at the intfc address */
3404 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3405 ip0->dst_address.as_u32)))
3410 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3411 format_ip4_address, &key0.ext_host_addr,
3412 clib_net_to_host_u16 (key0.ext_host_port),
3413 format_ip4_address, &out_addr,
3414 clib_net_to_host_u16 (key0.out_port),
3415 format_ip4_address, &new_addr0);
3416 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3417 next0 = SNAT_OUT2IN_NEXT_DROP;
3421 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
3422 !icmp_is_error_message (icmp0)))
3424 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
3425 next0 = SNAT_OUT2IN_NEXT_DROP;
3432 *p_proto = protocol;
3435 p_value->addr = new_addr0;
3436 p_value->fib_index = sm->inside_fib_index;
3437 p_value->port = ses0->in_port;
3439 *p_dont_translate = dont_translate;
3441 *(snat_det_session_t**)d = ses0;
3443 *(snat_det_map_t**)e = dm0;
3447 /**********************/
3448 /*** worker handoff ***/
3449 /**********************/
3451 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
3452 vlib_node_runtime_t * node,
3453 vlib_frame_t * frame)
3455 snat_main_t *sm = &snat_main;
3456 vlib_thread_main_t *tm = vlib_get_thread_main ();
3457 u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
3458 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3459 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3461 vlib_frame_queue_elt_t *hf = 0;
3462 vlib_frame_queue_t *fq;
3463 vlib_frame_t *f = 0;
3465 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3466 u32 next_worker_index = 0;
3467 u32 current_worker_index = ~0;
3468 u32 thread_index = vm->thread_index;
3469 vlib_frame_t *d = 0;
3471 ASSERT (vec_len (sm->workers));
3473 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3475 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3477 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3478 tm->n_vlib_mains - 1,
3479 (vlib_frame_queue_t *) (~0));
3482 from = vlib_frame_vector_args (frame);
3483 n_left_from = frame->n_vectors;
3485 while (n_left_from > 0)
3498 b0 = vlib_get_buffer (vm, bi0);
3500 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3501 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3503 ip0 = vlib_buffer_get_current (b0);
3505 next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
3507 if (PREDICT_FALSE (next_worker_index != thread_index))
3511 if (next_worker_index != current_worker_index)
3513 fq = is_vlib_frame_queue_congested (
3514 sm->fq_out2in_index, next_worker_index, NAT_FQ_NELTS - 2,
3515 congested_handoff_queue_by_worker_index);
3519 /* if this is 1st frame */
3522 d = vlib_get_frame_to_node (vm, sm->error_node_index);
3523 to_next_drop = vlib_frame_vector_args (d);
3526 to_next_drop[0] = bi0;
3529 b0->error = node->errors[SNAT_OUT2IN_ERROR_FQ_CONGESTED];
3534 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3536 hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
3538 handoff_queue_elt_by_worker_index);
3540 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3541 to_next_worker = &hf->buffer_index[hf->n_vectors];
3542 current_worker_index = next_worker_index;
3545 /* enqueue to correct worker thread */
3546 to_next_worker[0] = bi0;
3548 n_left_to_next_worker--;
3550 if (n_left_to_next_worker == 0)
3552 hf->n_vectors = VLIB_FRAME_SIZE;
3553 vlib_put_frame_queue_elt (hf);
3554 current_worker_index = ~0;
3555 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3562 /* if this is 1st frame */
3565 f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
3566 to_next = vlib_frame_vector_args (f);
3575 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3576 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3578 snat_out2in_worker_handoff_trace_t *t =
3579 vlib_add_trace (vm, node, b0, sizeof (*t));
3580 t->next_worker_index = next_worker_index;
3581 t->do_handoff = do_handoff;
3586 vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
3589 vlib_put_frame_to_node (vm, sm->error_node_index, d);
3592 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3594 /* Ship frames to the worker nodes */
3595 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3597 if (handoff_queue_elt_by_worker_index[i])
3599 hf = handoff_queue_elt_by_worker_index[i];
3601 * It works better to let the handoff node
3602 * rate-adapt, always ship the handoff queue element.
3604 if (1 || hf->n_vectors == hf->last_n_vectors)
3606 vlib_put_frame_queue_elt (hf);
3607 handoff_queue_elt_by_worker_index[i] = 0;
3610 hf->last_n_vectors = hf->n_vectors;
3612 congested_handoff_queue_by_worker_index[i] =
3613 (vlib_frame_queue_t *) (~0);
3616 current_worker_index = ~0;
3617 return frame->n_vectors;
3620 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
3621 .function = snat_out2in_worker_handoff_fn,
3622 .name = "nat44-out2in-worker-handoff",
3623 .vector_size = sizeof (u32),
3624 .format_trace = format_snat_out2in_worker_handoff_trace,
3625 .type = VLIB_NODE_TYPE_INTERNAL,
3627 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3628 .error_strings = snat_out2in_error_strings,
3637 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
3640 snat_out2in_fast_node_fn (vlib_main_t * vm,
3641 vlib_node_runtime_t * node,
3642 vlib_frame_t * frame)
3644 u32 n_left_from, * from, * to_next;
3645 snat_out2in_next_t next_index;
3646 u32 pkts_processed = 0;
3647 snat_main_t * sm = &snat_main;
3649 from = vlib_frame_vector_args (frame);
3650 n_left_from = frame->n_vectors;
3651 next_index = node->cached_next_index;
3653 while (n_left_from > 0)
3657 vlib_get_next_frame (vm, node, next_index,
3658 to_next, n_left_to_next);
3660 while (n_left_from > 0 && n_left_to_next > 0)
3664 u32 next0 = SNAT_OUT2IN_NEXT_DROP;
3668 u32 new_addr0, old_addr0;
3669 u16 new_port0, old_port0;
3670 udp_header_t * udp0;
3671 tcp_header_t * tcp0;
3672 icmp46_header_t * icmp0;
3673 snat_session_key_t key0, sm0;
3677 /* speculatively enqueue b0 to the current next frame */
3683 n_left_to_next -= 1;
3685 b0 = vlib_get_buffer (vm, bi0);
3687 ip0 = vlib_buffer_get_current (b0);
3688 udp0 = ip4_next_header (ip0);
3689 tcp0 = (tcp_header_t *) udp0;
3690 icmp0 = (icmp46_header_t *) udp0;
3692 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3693 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3695 vnet_feature_next (&next0, b0);
3697 if (PREDICT_FALSE(ip0->ttl == 1))
3699 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3700 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3701 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3703 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3707 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3709 if (PREDICT_FALSE (proto0 == ~0))
3712 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3714 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3715 rx_fib_index0, node, next0, ~0, 0, 0);
3719 key0.addr = ip0->dst_address;
3720 key0.port = udp0->dst_port;
3721 key0.fib_index = rx_fib_index0;
3723 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
3725 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3729 new_addr0 = sm0.addr.as_u32;
3730 new_port0 = sm0.port;
3731 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3732 old_addr0 = ip0->dst_address.as_u32;
3733 ip0->dst_address.as_u32 = new_addr0;
3735 sum0 = ip0->checksum;
3736 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3738 dst_address /* changed member */);
3739 ip0->checksum = ip_csum_fold (sum0);
3741 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3743 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3745 old_port0 = tcp0->dst_port;
3746 tcp0->dst_port = new_port0;
3748 sum0 = tcp0->checksum;
3749 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3751 dst_address /* changed member */);
3753 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3754 ip4_header_t /* cheat */,
3755 length /* changed member */);
3756 tcp0->checksum = ip_csum_fold(sum0);
3760 old_port0 = udp0->dst_port;
3761 udp0->dst_port = new_port0;
3767 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3769 sum0 = tcp0->checksum;
3770 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3772 dst_address /* changed member */);
3774 tcp0->checksum = ip_csum_fold(sum0);
3780 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3781 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3783 snat_out2in_trace_t *t =
3784 vlib_add_trace (vm, node, b0, sizeof (*t));
3785 t->sw_if_index = sw_if_index0;
3786 t->next_index = next0;
3789 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3791 /* verify speculative enqueue, maybe switch current next frame */
3792 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3793 to_next, n_left_to_next,
3797 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3800 vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3801 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3803 return frame->n_vectors;
3806 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3807 .function = snat_out2in_fast_node_fn,
3808 .name = "nat44-out2in-fast",
3809 .vector_size = sizeof (u32),
3810 .format_trace = format_snat_out2in_fast_trace,
3811 .type = VLIB_NODE_TYPE_INTERNAL,
3813 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3814 .error_strings = snat_out2in_error_strings,
3816 .runtime_data_bytes = sizeof (snat_runtime_t),
3818 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3820 /* edit / add dispositions here */
3822 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3823 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3824 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3825 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3828 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);