2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vlib/vlib.h>
17 #include <vnet/vnet.h>
18 #include <vnet/pg/pg.h>
19 #include <vnet/handoff.h>
21 #include <vnet/ip/ip.h>
22 #include <vnet/udp/udp.h>
23 #include <vnet/ethernet/ethernet.h>
24 #include <vnet/fib/ip4_fib.h>
26 #include <nat/nat_ipfix_logging.h>
27 #include <nat/nat_det.h>
28 #include <nat/nat_reass.h>
29 #include <nat/nat_inlines.h>
31 #include <vppinfra/hash.h>
32 #include <vppinfra/error.h>
33 #include <vppinfra/elog.h>
39 } snat_out2in_trace_t;
42 u32 next_worker_index;
44 } snat_out2in_worker_handoff_trace_t;
46 /* packet trace format function */
47 static u8 * format_snat_out2in_trace (u8 * s, va_list * args)
49 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
50 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
51 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
53 s = format (s, "NAT44_OUT2IN: sw_if_index %d, next index %d, session index %d",
54 t->sw_if_index, t->next_index, t->session_index);
58 static u8 * format_snat_out2in_fast_trace (u8 * s, va_list * args)
60 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
61 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
62 snat_out2in_trace_t * t = va_arg (*args, snat_out2in_trace_t *);
64 s = format (s, "NAT44_OUT2IN_FAST: sw_if_index %d, next index %d",
65 t->sw_if_index, t->next_index);
69 static u8 * format_snat_out2in_worker_handoff_trace (u8 * s, va_list * args)
71 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
72 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
73 snat_out2in_worker_handoff_trace_t * t =
74 va_arg (*args, snat_out2in_worker_handoff_trace_t *);
77 m = t->do_handoff ? "next worker" : "same worker";
78 s = format (s, "NAT44_OUT2IN_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
87 } nat44_out2in_reass_trace_t;
89 static u8 * format_nat44_out2in_reass_trace (u8 * s, va_list * args)
91 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
92 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
93 nat44_out2in_reass_trace_t * t = va_arg (*args, nat44_out2in_reass_trace_t *);
95 s = format (s, "NAT44_OUT2IN_REASS: sw_if_index %d, next index %d, status %s",
96 t->sw_if_index, t->next_index,
97 t->cached ? "cached" : "translated");
102 vlib_node_registration_t snat_out2in_node;
103 vlib_node_registration_t snat_out2in_fast_node;
104 vlib_node_registration_t snat_out2in_worker_handoff_node;
105 vlib_node_registration_t snat_det_out2in_node;
106 vlib_node_registration_t nat44_out2in_reass_node;
107 vlib_node_registration_t nat44_ed_out2in_node;
108 vlib_node_registration_t nat44_ed_out2in_slowpath_node;
110 #define foreach_snat_out2in_error \
111 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol") \
112 _(OUT2IN_PACKETS, "Good out2in packets processed") \
113 _(OUT_OF_PORTS, "Out of ports") \
114 _(BAD_ICMP_TYPE, "unsupported ICMP type") \
115 _(NO_TRANSLATION, "No translation") \
116 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded") \
117 _(DROP_FRAGMENT, "Drop fragment") \
118 _(MAX_REASS, "Maximum reassemblies exceeded") \
119 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
120 _(FQ_CONGESTED, "Handoff frame queue congested")
123 #define _(sym,str) SNAT_OUT2IN_ERROR_##sym,
124 foreach_snat_out2in_error
127 } snat_out2in_error_t;
129 static char * snat_out2in_error_strings[] = {
130 #define _(sym,string) string,
131 foreach_snat_out2in_error
136 SNAT_OUT2IN_NEXT_DROP,
137 SNAT_OUT2IN_NEXT_LOOKUP,
138 SNAT_OUT2IN_NEXT_ICMP_ERROR,
139 SNAT_OUT2IN_NEXT_REASS,
141 } snat_out2in_next_t;
144 nat44_o2i_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void * arg)
146 snat_main_t *sm = &snat_main;
147 nat44_is_idle_session_ctx_t *ctx = arg;
149 u64 sess_timeout_time;
150 snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
152 clib_bihash_kv_8_8_t s_kv;
154 s = pool_elt_at_index (tsm->sessions, kv->value);
155 sess_timeout_time = s->last_heard + (f64)nat44_session_get_timeout(sm, s);
156 if (ctx->now >= sess_timeout_time)
158 s_kv.key = s->in2out.as_u64;
159 if (clib_bihash_add_del_8_8 (&tsm->in2out, &s_kv, 0))
160 nat_log_warn ("out2in key del failed");
162 snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
163 s->out2in.addr.as_u32,
167 s->in2out.fib_index);
169 if (!snat_is_session_static (s))
170 snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
173 nat44_delete_session (sm, s, ctx->thread_index);
181 * @brief Create session for static mapping.
183 * Create NAT session initiated by host from external network with static
186 * @param sm NAT main.
187 * @param b0 Vlib buffer.
188 * @param in2out In2out NAT44 session key.
189 * @param out2in Out2in NAT44 session key.
190 * @param node Vlib node.
192 * @returns SNAT session if successfully created otherwise 0.
194 static inline snat_session_t *
195 create_session_for_static_mapping (snat_main_t *sm,
197 snat_session_key_t in2out,
198 snat_session_key_t out2in,
199 vlib_node_runtime_t * node,
205 clib_bihash_kv_8_8_t kv0;
208 nat44_is_idle_session_ctx_t ctx0;
210 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
212 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
213 nat_log_notice ("maximum sessions exceeded");
217 ip0 = vlib_buffer_get_current (b0);
218 udp0 = ip4_next_header (ip0);
220 u = nat_user_get_or_create (sm, &in2out.addr, in2out.fib_index, thread_index);
223 nat_log_warn ("create NAT user failed");
227 s = nat_session_alloc_or_recycle (sm, u, thread_index);
230 nat44_delete_user_with_no_session (sm, u, thread_index);
231 nat_log_warn ("create NAT session failed");
235 s->outside_address_index = ~0;
236 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
237 s->ext_host_addr.as_u32 = ip0->src_address.as_u32;
238 s->ext_host_port = udp0->src_port;
239 user_session_increment (sm, u, 1 /* static */);
242 s->in2out.protocol = out2in.protocol;
244 /* Add to translation hashes */
246 ctx0.thread_index = thread_index;
247 kv0.key = s->in2out.as_u64;
248 kv0.value = s - sm->per_thread_data[thread_index].sessions;
249 if (clib_bihash_add_or_overwrite_stale_8_8 (
250 &sm->per_thread_data[thread_index].in2out, &kv0,
251 nat44_i2o_is_idle_session_cb, &ctx0))
252 nat_log_notice ("in2out key add failed");
254 kv0.key = s->out2in.as_u64;
256 if (clib_bihash_add_or_overwrite_stale_8_8 (
257 &sm->per_thread_data[thread_index].out2in, &kv0,
258 nat44_o2i_is_idle_session_cb, &ctx0))
259 nat_log_notice ("out2in key add failed");
262 snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
263 s->out2in.addr.as_u32,
267 s->in2out.fib_index);
272 snat_out2in_error_t icmp_get_key(ip4_header_t *ip0,
273 snat_session_key_t *p_key0)
275 icmp46_header_t *icmp0;
276 snat_session_key_t key0;
277 icmp_echo_header_t *echo0, *inner_echo0 = 0;
278 ip4_header_t *inner_ip0;
280 icmp46_header_t *inner_icmp0;
282 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
283 echo0 = (icmp_echo_header_t *)(icmp0+1);
285 if (!icmp_is_error_message (icmp0))
287 key0.protocol = SNAT_PROTOCOL_ICMP;
288 key0.addr = ip0->dst_address;
289 key0.port = echo0->identifier;
293 inner_ip0 = (ip4_header_t *)(echo0+1);
294 l4_header = ip4_next_header (inner_ip0);
295 key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
296 key0.addr = inner_ip0->src_address;
297 switch (key0.protocol)
299 case SNAT_PROTOCOL_ICMP:
300 inner_icmp0 = (icmp46_header_t*)l4_header;
301 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
302 key0.port = inner_echo0->identifier;
304 case SNAT_PROTOCOL_UDP:
305 case SNAT_PROTOCOL_TCP:
306 key0.port = ((tcp_udp_header_t*)l4_header)->src_port;
309 return SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL;
313 return -1; /* success */
317 * Get address and port values to be used for ICMP packet translation
318 * and create session if needed
320 * @param[in,out] sm NAT main
321 * @param[in,out] node NAT node runtime
322 * @param[in] thread_index thread index
323 * @param[in,out] b0 buffer containing packet to be translated
324 * @param[out] p_proto protocol used for matching
325 * @param[out] p_value address and port after NAT translation
326 * @param[out] p_dont_translate if packet should not be translated
327 * @param d optional parameter
328 * @param e optional parameter
330 u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node,
331 u32 thread_index, vlib_buffer_t *b0,
332 ip4_header_t *ip0, u8 *p_proto,
333 snat_session_key_t *p_value,
334 u8 *p_dont_translate, void *d, void *e)
336 icmp46_header_t *icmp0;
339 snat_session_key_t key0;
340 snat_session_key_t sm0;
341 snat_session_t *s0 = 0;
342 u8 dont_translate = 0;
343 clib_bihash_kv_8_8_t kv0, value0;
348 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
349 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
350 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
354 err = icmp_get_key (ip0, &key0);
357 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
358 next0 = SNAT_OUT2IN_NEXT_DROP;
361 key0.fib_index = rx_fib_index0;
363 kv0.key = key0.as_u64;
365 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
368 /* Try to match static mapping by external address and port,
369 destination address and port in packet */
370 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
372 if (!sm->forwarding_enabled)
374 /* Don't NAT packet aimed at the intfc address */
375 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
376 ip0->dst_address.as_u32)))
381 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
382 next0 = SNAT_OUT2IN_NEXT_DROP;
392 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
393 (icmp0->type != ICMP4_echo_request || !is_addr_only)))
395 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
396 next0 = SNAT_OUT2IN_NEXT_DROP;
400 /* Create session initiated by host from external network */
401 s0 = create_session_for_static_mapping(sm, b0, sm0, key0,
403 vlib_time_now (sm->vlib_main));
407 next0 = SNAT_OUT2IN_NEXT_DROP;
413 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
414 icmp0->type != ICMP4_echo_request &&
415 !icmp_is_error_message (icmp0)))
417 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
418 next0 = SNAT_OUT2IN_NEXT_DROP;
422 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
427 *p_proto = key0.protocol;
429 *p_value = s0->in2out;
430 *p_dont_translate = dont_translate;
432 *(snat_session_t**)d = s0;
437 * Get address and port values to be used for ICMP packet translation
439 * @param[in] sm NAT main
440 * @param[in,out] node NAT node runtime
441 * @param[in] thread_index thread index
442 * @param[in,out] b0 buffer containing packet to be translated
443 * @param[out] p_proto protocol used for matching
444 * @param[out] p_value address and port after NAT translation
445 * @param[out] p_dont_translate if packet should not be translated
446 * @param d optional parameter
447 * @param e optional parameter
449 u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node,
450 u32 thread_index, vlib_buffer_t *b0,
451 ip4_header_t *ip0, u8 *p_proto,
452 snat_session_key_t *p_value,
453 u8 *p_dont_translate, void *d, void *e)
455 icmp46_header_t *icmp0;
458 snat_session_key_t key0;
459 snat_session_key_t sm0;
460 u8 dont_translate = 0;
465 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
466 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
467 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
469 err = icmp_get_key (ip0, &key0);
472 b0->error = node->errors[err];
473 next0 = SNAT_OUT2IN_NEXT_DROP;
476 key0.fib_index = rx_fib_index0;
478 if (snat_static_mapping_match(sm, key0, &sm0, 1, &is_addr_only, 0, 0))
480 /* Don't NAT packet aimed at the intfc address */
481 if (is_interface_addr(sm, node, sw_if_index0, ip0->dst_address.as_u32))
486 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
487 next0 = SNAT_OUT2IN_NEXT_DROP;
491 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
492 (icmp0->type != ICMP4_echo_request || !is_addr_only) &&
493 !icmp_is_error_message (icmp0)))
495 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
496 next0 = SNAT_OUT2IN_NEXT_DROP;
503 *p_proto = key0.protocol;
504 *p_dont_translate = dont_translate;
508 static inline u32 icmp_out2in (snat_main_t *sm,
511 icmp46_header_t * icmp0,
514 vlib_node_runtime_t * node,
520 snat_session_key_t sm0;
522 icmp_echo_header_t *echo0, *inner_echo0 = 0;
523 ip4_header_t *inner_ip0 = 0;
525 icmp46_header_t *inner_icmp0;
527 u32 new_addr0, old_addr0;
528 u16 old_id0, new_id0;
533 echo0 = (icmp_echo_header_t *)(icmp0+1);
535 next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, ip0,
536 &protocol, &sm0, &dont_translate, d, e);
539 if (next0 == SNAT_OUT2IN_NEXT_DROP || dont_translate)
542 sum0 = ip_incremental_checksum (0, icmp0,
543 ntohs(ip0->length) - ip4_header_bytes (ip0));
544 checksum0 = ~ip_csum_fold (sum0);
545 if (checksum0 != 0 && checksum0 != 0xffff)
547 next0 = SNAT_OUT2IN_NEXT_DROP;
551 old_addr0 = ip0->dst_address.as_u32;
552 new_addr0 = ip0->dst_address.as_u32 = sm0.addr.as_u32;
553 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
555 sum0 = ip0->checksum;
556 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
557 dst_address /* changed member */);
558 ip0->checksum = ip_csum_fold (sum0);
560 if (icmp0->checksum == 0)
561 icmp0->checksum = 0xffff;
563 if (!icmp_is_error_message (icmp0))
566 if (PREDICT_FALSE(new_id0 != echo0->identifier))
568 old_id0 = echo0->identifier;
570 echo0->identifier = new_id0;
572 sum0 = icmp0->checksum;
573 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
574 identifier /* changed member */);
575 icmp0->checksum = ip_csum_fold (sum0);
580 inner_ip0 = (ip4_header_t *)(echo0+1);
581 l4_header = ip4_next_header (inner_ip0);
583 if (!ip4_header_checksum_is_valid (inner_ip0))
585 next0 = SNAT_OUT2IN_NEXT_DROP;
589 old_addr0 = inner_ip0->src_address.as_u32;
590 inner_ip0->src_address = sm0.addr;
591 new_addr0 = inner_ip0->src_address.as_u32;
593 sum0 = icmp0->checksum;
594 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
595 src_address /* changed member */);
596 icmp0->checksum = ip_csum_fold (sum0);
600 case SNAT_PROTOCOL_ICMP:
601 inner_icmp0 = (icmp46_header_t*)l4_header;
602 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
604 old_id0 = inner_echo0->identifier;
606 inner_echo0->identifier = new_id0;
608 sum0 = icmp0->checksum;
609 sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
611 icmp0->checksum = ip_csum_fold (sum0);
613 case SNAT_PROTOCOL_UDP:
614 case SNAT_PROTOCOL_TCP:
615 old_id0 = ((tcp_udp_header_t*)l4_header)->src_port;
617 ((tcp_udp_header_t*)l4_header)->src_port = new_id0;
619 sum0 = icmp0->checksum;
620 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
622 icmp0->checksum = ip_csum_fold (sum0);
634 static inline u32 icmp_out2in_slow_path (snat_main_t *sm,
637 icmp46_header_t * icmp0,
640 vlib_node_runtime_t * node,
643 snat_session_t ** p_s0)
645 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
646 next0, thread_index, p_s0, 0);
647 snat_session_t * s0 = *p_s0;
648 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
651 nat44_session_update_counters (s0, now,
652 vlib_buffer_length_in_chain (sm->vlib_main, b0));
653 /* Per-user LRU list maintenance */
654 nat44_session_update_lru (sm, s0, thread_index);
660 nat_out2in_sm_unknown_proto (snat_main_t *sm,
665 clib_bihash_kv_8_8_t kv, value;
666 snat_static_mapping_t *m;
667 snat_session_key_t m_key;
668 u32 old_addr, new_addr;
671 m_key.addr = ip->dst_address;
675 kv.key = m_key.as_u64;
676 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
679 m = pool_elt_at_index (sm->static_mappings, value.value);
681 old_addr = ip->dst_address.as_u32;
682 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
684 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
685 ip->checksum = ip_csum_fold (sum);
687 vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
692 snat_out2in_node_fn (vlib_main_t * vm,
693 vlib_node_runtime_t * node,
694 vlib_frame_t * frame)
696 u32 n_left_from, * from, * to_next;
697 snat_out2in_next_t next_index;
698 u32 pkts_processed = 0;
699 snat_main_t * sm = &snat_main;
700 f64 now = vlib_time_now (vm);
701 u32 thread_index = vm->thread_index;
703 from = vlib_frame_vector_args (frame);
704 n_left_from = frame->n_vectors;
705 next_index = node->cached_next_index;
707 while (n_left_from > 0)
711 vlib_get_next_frame (vm, node, next_index,
712 to_next, n_left_to_next);
714 while (n_left_from >= 4 && n_left_to_next >= 2)
717 vlib_buffer_t * b0, * b1;
718 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
719 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
720 u32 sw_if_index0, sw_if_index1;
721 ip4_header_t * ip0, *ip1;
722 ip_csum_t sum0, sum1;
723 u32 new_addr0, old_addr0;
724 u16 new_port0, old_port0;
725 u32 new_addr1, old_addr1;
726 u16 new_port1, old_port1;
727 udp_header_t * udp0, * udp1;
728 tcp_header_t * tcp0, * tcp1;
729 icmp46_header_t * icmp0, * icmp1;
730 snat_session_key_t key0, key1, sm0, sm1;
731 u32 rx_fib_index0, rx_fib_index1;
733 snat_session_t * s0 = 0, * s1 = 0;
734 clib_bihash_kv_8_8_t kv0, kv1, value0, value1;
736 /* Prefetch next iteration. */
738 vlib_buffer_t * p2, * p3;
740 p2 = vlib_get_buffer (vm, from[2]);
741 p3 = vlib_get_buffer (vm, from[3]);
743 vlib_prefetch_buffer_header (p2, LOAD);
744 vlib_prefetch_buffer_header (p3, LOAD);
746 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
747 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
750 /* speculatively enqueue b0 and b1 to the current next frame */
751 to_next[0] = bi0 = from[0];
752 to_next[1] = bi1 = from[1];
758 b0 = vlib_get_buffer (vm, bi0);
759 b1 = vlib_get_buffer (vm, bi1);
761 vnet_buffer (b0)->snat.flags = 0;
762 vnet_buffer (b1)->snat.flags = 0;
764 ip0 = vlib_buffer_get_current (b0);
765 udp0 = ip4_next_header (ip0);
766 tcp0 = (tcp_header_t *) udp0;
767 icmp0 = (icmp46_header_t *) udp0;
769 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
770 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
773 if (PREDICT_FALSE(ip0->ttl == 1))
775 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
776 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
777 ICMP4_time_exceeded_ttl_exceeded_in_transit,
779 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
783 proto0 = ip_proto_to_snat_proto (ip0->protocol);
785 if (PREDICT_FALSE (proto0 == ~0))
787 if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
789 if (!sm->forwarding_enabled)
791 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
792 next0 = SNAT_OUT2IN_NEXT_DROP;
798 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
800 next0 = icmp_out2in_slow_path
801 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
802 next0, now, thread_index, &s0);
806 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
808 next0 = SNAT_OUT2IN_NEXT_REASS;
812 key0.addr = ip0->dst_address;
813 key0.port = udp0->dst_port;
814 key0.protocol = proto0;
815 key0.fib_index = rx_fib_index0;
817 kv0.key = key0.as_u64;
819 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
822 /* Try to match static mapping by external address and port,
823 destination address and port in packet */
824 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
827 * Send DHCP packets to the ipv4 stack, or we won't
828 * be able to use dhcp client on the outside interface
830 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
831 && (udp0->dst_port ==
832 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
834 vnet_feature_next (&next0, b0);
838 if (!sm->forwarding_enabled)
840 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
841 next0 = SNAT_OUT2IN_NEXT_DROP;
846 /* Create session initiated by host from external network */
847 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
851 next0 = SNAT_OUT2IN_NEXT_DROP;
856 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
859 old_addr0 = ip0->dst_address.as_u32;
860 ip0->dst_address = s0->in2out.addr;
861 new_addr0 = ip0->dst_address.as_u32;
862 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
864 sum0 = ip0->checksum;
865 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
867 dst_address /* changed member */);
868 ip0->checksum = ip_csum_fold (sum0);
870 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
872 old_port0 = tcp0->dst_port;
873 tcp0->dst_port = s0->in2out.port;
874 new_port0 = tcp0->dst_port;
876 sum0 = tcp0->checksum;
877 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
879 dst_address /* changed member */);
881 sum0 = ip_csum_update (sum0, old_port0, new_port0,
882 ip4_header_t /* cheat */,
883 length /* changed member */);
884 tcp0->checksum = ip_csum_fold(sum0);
888 old_port0 = udp0->dst_port;
889 udp0->dst_port = s0->in2out.port;
894 nat44_session_update_counters (s0, now,
895 vlib_buffer_length_in_chain (vm, b0));
896 /* Per-user LRU list maintenance */
897 nat44_session_update_lru (sm, s0, thread_index);
900 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
901 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
903 snat_out2in_trace_t *t =
904 vlib_add_trace (vm, node, b0, sizeof (*t));
905 t->sw_if_index = sw_if_index0;
906 t->next_index = next0;
907 t->session_index = ~0;
909 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
912 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
915 ip1 = vlib_buffer_get_current (b1);
916 udp1 = ip4_next_header (ip1);
917 tcp1 = (tcp_header_t *) udp1;
918 icmp1 = (icmp46_header_t *) udp1;
920 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
921 rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
924 if (PREDICT_FALSE(ip1->ttl == 1))
926 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
927 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
928 ICMP4_time_exceeded_ttl_exceeded_in_transit,
930 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
934 proto1 = ip_proto_to_snat_proto (ip1->protocol);
936 if (PREDICT_FALSE (proto1 == ~0))
938 if (nat_out2in_sm_unknown_proto(sm, b1, ip1, rx_fib_index1))
940 if (!sm->forwarding_enabled)
942 b1->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
943 next1 = SNAT_OUT2IN_NEXT_DROP;
949 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
951 next1 = icmp_out2in_slow_path
952 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
953 next1, now, thread_index, &s1);
957 if (PREDICT_FALSE (ip4_is_fragment (ip1)))
959 next1 = SNAT_OUT2IN_NEXT_REASS;
963 key1.addr = ip1->dst_address;
964 key1.port = udp1->dst_port;
965 key1.protocol = proto1;
966 key1.fib_index = rx_fib_index1;
968 kv1.key = key1.as_u64;
970 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
973 /* Try to match static mapping by external address and port,
974 destination address and port in packet */
975 if (snat_static_mapping_match(sm, key1, &sm1, 1, 0, 0, 0))
978 * Send DHCP packets to the ipv4 stack, or we won't
979 * be able to use dhcp client on the outside interface
981 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
982 && (udp1->dst_port ==
983 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
985 vnet_feature_next (&next1, b1);
989 if (!sm->forwarding_enabled)
991 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
992 next1 = SNAT_OUT2IN_NEXT_DROP;
997 /* Create session initiated by host from external network */
998 s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node,
1002 next1 = SNAT_OUT2IN_NEXT_DROP;
1007 s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1010 old_addr1 = ip1->dst_address.as_u32;
1011 ip1->dst_address = s1->in2out.addr;
1012 new_addr1 = ip1->dst_address.as_u32;
1013 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
1015 sum1 = ip1->checksum;
1016 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1018 dst_address /* changed member */);
1019 ip1->checksum = ip_csum_fold (sum1);
1021 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1023 old_port1 = tcp1->dst_port;
1024 tcp1->dst_port = s1->in2out.port;
1025 new_port1 = tcp1->dst_port;
1027 sum1 = tcp1->checksum;
1028 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1030 dst_address /* changed member */);
1032 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1033 ip4_header_t /* cheat */,
1034 length /* changed member */);
1035 tcp1->checksum = ip_csum_fold(sum1);
1039 old_port1 = udp1->dst_port;
1040 udp1->dst_port = s1->in2out.port;
1045 nat44_session_update_counters (s1, now,
1046 vlib_buffer_length_in_chain (vm, b1));
1047 /* Per-user LRU list maintenance */
1048 nat44_session_update_lru (sm, s1, thread_index);
1051 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1052 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1054 snat_out2in_trace_t *t =
1055 vlib_add_trace (vm, node, b1, sizeof (*t));
1056 t->sw_if_index = sw_if_index1;
1057 t->next_index = next1;
1058 t->session_index = ~0;
1060 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1063 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
1065 /* verify speculative enqueues, maybe switch current next frame */
1066 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1067 to_next, n_left_to_next,
1068 bi0, bi1, next0, next1);
1071 while (n_left_from > 0 && n_left_to_next > 0)
1075 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1079 u32 new_addr0, old_addr0;
1080 u16 new_port0, old_port0;
1081 udp_header_t * udp0;
1082 tcp_header_t * tcp0;
1083 icmp46_header_t * icmp0;
1084 snat_session_key_t key0, sm0;
1087 snat_session_t * s0 = 0;
1088 clib_bihash_kv_8_8_t kv0, value0;
1090 /* speculatively enqueue b0 to the current next frame */
1096 n_left_to_next -= 1;
1098 b0 = vlib_get_buffer (vm, bi0);
1100 vnet_buffer (b0)->snat.flags = 0;
1102 ip0 = vlib_buffer_get_current (b0);
1103 udp0 = ip4_next_header (ip0);
1104 tcp0 = (tcp_header_t *) udp0;
1105 icmp0 = (icmp46_header_t *) udp0;
1107 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1108 rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1111 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1113 if (PREDICT_FALSE (proto0 == ~0))
1115 if (nat_out2in_sm_unknown_proto(sm, b0, ip0, rx_fib_index0))
1117 if (!sm->forwarding_enabled)
1119 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1120 next0 = SNAT_OUT2IN_NEXT_DROP;
1126 if (PREDICT_FALSE(ip0->ttl == 1))
1128 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1129 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1130 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1132 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
1136 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1138 next0 = icmp_out2in_slow_path
1139 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1140 next0, now, thread_index, &s0);
1144 if (PREDICT_FALSE (ip4_is_fragment (ip0)))
1146 next0 = SNAT_OUT2IN_NEXT_REASS;
1150 key0.addr = ip0->dst_address;
1151 key0.port = udp0->dst_port;
1152 key0.protocol = proto0;
1153 key0.fib_index = rx_fib_index0;
1155 kv0.key = key0.as_u64;
1157 if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in,
1160 /* Try to match static mapping by external address and port,
1161 destination address and port in packet */
1162 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1165 * Send DHCP packets to the ipv4 stack, or we won't
1166 * be able to use dhcp client on the outside interface
1168 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1169 && (udp0->dst_port ==
1170 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1172 vnet_feature_next (&next0, b0);
1176 if (!sm->forwarding_enabled)
1178 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1179 next0 = SNAT_OUT2IN_NEXT_DROP;
1184 /* Create session initiated by host from external network */
1185 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1189 next0 = SNAT_OUT2IN_NEXT_DROP;
1194 s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1197 old_addr0 = ip0->dst_address.as_u32;
1198 ip0->dst_address = s0->in2out.addr;
1199 new_addr0 = ip0->dst_address.as_u32;
1200 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1202 sum0 = ip0->checksum;
1203 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1205 dst_address /* changed member */);
1206 ip0->checksum = ip_csum_fold (sum0);
1208 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1210 old_port0 = tcp0->dst_port;
1211 tcp0->dst_port = s0->in2out.port;
1212 new_port0 = tcp0->dst_port;
1214 sum0 = tcp0->checksum;
1215 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1217 dst_address /* changed member */);
1219 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1220 ip4_header_t /* cheat */,
1221 length /* changed member */);
1222 tcp0->checksum = ip_csum_fold(sum0);
1226 old_port0 = udp0->dst_port;
1227 udp0->dst_port = s0->in2out.port;
1232 nat44_session_update_counters (s0, now,
1233 vlib_buffer_length_in_chain (vm, b0));
1234 /* Per-user LRU list maintenance */
1235 nat44_session_update_lru (sm, s0, thread_index);
1238 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1239 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1241 snat_out2in_trace_t *t =
1242 vlib_add_trace (vm, node, b0, sizeof (*t));
1243 t->sw_if_index = sw_if_index0;
1244 t->next_index = next0;
1245 t->session_index = ~0;
1247 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1250 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1252 /* verify speculative enqueue, maybe switch current next frame */
1253 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1254 to_next, n_left_to_next,
1258 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1261 vlib_node_increment_counter (vm, snat_out2in_node.index,
1262 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1264 return frame->n_vectors;
1267 VLIB_REGISTER_NODE (snat_out2in_node) = {
1268 .function = snat_out2in_node_fn,
1269 .name = "nat44-out2in",
1270 .vector_size = sizeof (u32),
1271 .format_trace = format_snat_out2in_trace,
1272 .type = VLIB_NODE_TYPE_INTERNAL,
1274 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1275 .error_strings = snat_out2in_error_strings,
1277 .runtime_data_bytes = sizeof (snat_runtime_t),
1279 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1281 /* edit / add dispositions here */
1283 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1284 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1285 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1286 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1289 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_node, snat_out2in_node_fn);
1292 nat44_out2in_reass_node_fn (vlib_main_t * vm,
1293 vlib_node_runtime_t * node,
1294 vlib_frame_t * frame)
1296 u32 n_left_from, *from, *to_next;
1297 snat_out2in_next_t next_index;
1298 u32 pkts_processed = 0;
1299 snat_main_t *sm = &snat_main;
1300 f64 now = vlib_time_now (vm);
1301 u32 thread_index = vm->thread_index;
1302 snat_main_per_thread_data_t *per_thread_data =
1303 &sm->per_thread_data[thread_index];
1304 u32 *fragments_to_drop = 0;
1305 u32 *fragments_to_loopback = 0;
1307 from = vlib_frame_vector_args (frame);
1308 n_left_from = frame->n_vectors;
1309 next_index = node->cached_next_index;
1311 while (n_left_from > 0)
1315 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1317 while (n_left_from > 0 && n_left_to_next > 0)
1319 u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
1324 nat_reass_ip4_t *reass0;
1325 udp_header_t * udp0;
1326 tcp_header_t * tcp0;
1327 snat_session_key_t key0, sm0;
1328 clib_bihash_kv_8_8_t kv0, value0;
1329 snat_session_t * s0 = 0;
1330 u16 old_port0, new_port0;
1333 /* speculatively enqueue b0 to the current next frame */
1339 n_left_to_next -= 1;
1341 b0 = vlib_get_buffer (vm, bi0);
1342 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
1344 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1345 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
1348 if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
1350 next0 = SNAT_OUT2IN_NEXT_DROP;
1351 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
1355 ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
1356 udp0 = ip4_next_header (ip0);
1357 tcp0 = (tcp_header_t *) udp0;
1358 proto0 = ip_proto_to_snat_proto (ip0->protocol);
1360 reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
1365 &fragments_to_drop);
1367 if (PREDICT_FALSE (!reass0))
1369 next0 = SNAT_OUT2IN_NEXT_DROP;
1370 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_REASS];
1371 nat_log_notice ("maximum reassemblies exceeded");
1375 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1377 key0.addr = ip0->dst_address;
1378 key0.port = udp0->dst_port;
1379 key0.protocol = proto0;
1380 key0.fib_index = rx_fib_index0;
1381 kv0.key = key0.as_u64;
1383 if (clib_bihash_search_8_8 (&per_thread_data->out2in, &kv0, &value0))
1385 /* Try to match static mapping by external address and port,
1386 destination address and port in packet */
1387 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1390 * Send DHCP packets to the ipv4 stack, or we won't
1391 * be able to use dhcp client on the outside interface
1393 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
1395 == clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
1397 vnet_feature_next (&next0, b0);
1401 if (!sm->forwarding_enabled)
1403 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1404 next0 = SNAT_OUT2IN_NEXT_DROP;
1409 /* Create session initiated by host from external network */
1410 s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node,
1414 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
1415 next0 = SNAT_OUT2IN_NEXT_DROP;
1418 reass0->sess_index = s0 - per_thread_data->sessions;
1419 reass0->thread_index = thread_index;
1423 s0 = pool_elt_at_index (per_thread_data->sessions,
1425 reass0->sess_index = value0.value;
1427 nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
1431 if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
1433 if (nat_ip4_reass_add_fragment (reass0, bi0))
1435 b0->error = node->errors[SNAT_OUT2IN_ERROR_MAX_FRAG];
1436 nat_log_notice ("maximum fragments per reassembly exceeded");
1437 next0 = SNAT_OUT2IN_NEXT_DROP;
1443 s0 = pool_elt_at_index (per_thread_data->sessions,
1444 reass0->sess_index);
1447 old_addr0 = ip0->dst_address.as_u32;
1448 ip0->dst_address = s0->in2out.addr;
1449 new_addr0 = ip0->dst_address.as_u32;
1450 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1452 sum0 = ip0->checksum;
1453 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1455 dst_address /* changed member */);
1456 ip0->checksum = ip_csum_fold (sum0);
1458 if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
1460 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1462 old_port0 = tcp0->dst_port;
1463 tcp0->dst_port = s0->in2out.port;
1464 new_port0 = tcp0->dst_port;
1466 sum0 = tcp0->checksum;
1467 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1469 dst_address /* changed member */);
1471 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1472 ip4_header_t /* cheat */,
1473 length /* changed member */);
1474 tcp0->checksum = ip_csum_fold(sum0);
1478 old_port0 = udp0->dst_port;
1479 udp0->dst_port = s0->in2out.port;
1485 nat44_session_update_counters (s0, now,
1486 vlib_buffer_length_in_chain (vm, b0));
1487 /* Per-user LRU list maintenance */
1488 nat44_session_update_lru (sm, s0, thread_index);
1491 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1492 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1494 nat44_out2in_reass_trace_t *t =
1495 vlib_add_trace (vm, node, b0, sizeof (*t));
1496 t->cached = cached0;
1497 t->sw_if_index = sw_if_index0;
1498 t->next_index = next0;
1508 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
1510 /* verify speculative enqueue, maybe switch current next frame */
1511 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1512 to_next, n_left_to_next,
1516 if (n_left_from == 0 && vec_len (fragments_to_loopback))
1518 from = vlib_frame_vector_args (frame);
1519 u32 len = vec_len (fragments_to_loopback);
1520 if (len <= VLIB_FRAME_SIZE)
1522 clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
1524 vec_reset_length (fragments_to_loopback);
1529 fragments_to_loopback + (len - VLIB_FRAME_SIZE),
1530 sizeof (u32) * VLIB_FRAME_SIZE);
1531 n_left_from = VLIB_FRAME_SIZE;
1532 _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
1537 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1540 vlib_node_increment_counter (vm, nat44_out2in_reass_node.index,
1541 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
1544 nat_send_all_to_node (vm, fragments_to_drop, node,
1545 &node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT],
1546 SNAT_OUT2IN_NEXT_DROP);
1548 vec_free (fragments_to_drop);
1549 vec_free (fragments_to_loopback);
1550 return frame->n_vectors;
1553 VLIB_REGISTER_NODE (nat44_out2in_reass_node) = {
1554 .function = nat44_out2in_reass_node_fn,
1555 .name = "nat44-out2in-reass",
1556 .vector_size = sizeof (u32),
1557 .format_trace = format_nat44_out2in_reass_trace,
1558 .type = VLIB_NODE_TYPE_INTERNAL,
1560 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
1561 .error_strings = snat_out2in_error_strings,
1563 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
1565 /* edit / add dispositions here */
1567 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
1568 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
1569 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1570 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
1573 VLIB_NODE_FUNCTION_MULTIARCH (nat44_out2in_reass_node,
1574 nat44_out2in_reass_node_fn);
1576 /*******************************/
1577 /*** endpoint-dependent mode ***/
1578 /*******************************/
1580 NAT44_ED_OUT2IN_NEXT_DROP,
1581 NAT44_ED_OUT2IN_NEXT_LOOKUP,
1582 NAT44_ED_OUT2IN_NEXT_ICMP_ERROR,
1583 NAT44_ED_OUT2IN_NEXT_IN2OUT,
1584 NAT44_ED_OUT2IN_NEXT_SLOW_PATH,
1585 NAT44_ED_OUT2IN_N_NEXT,
1586 } nat44_ed_out2in_next_t;
1593 } nat44_ed_out2in_trace_t;
1596 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
1598 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1599 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1600 nat44_ed_out2in_trace_t *t = va_arg (*args, nat44_ed_out2in_trace_t *);
1603 tag = t->is_slow_path ? "NAT44_OUT2IN_SLOW_PATH" : "NAT44_OUT2IN_FAST_PATH";
1605 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
1606 t->sw_if_index, t->next_index, t->session_index);
1612 icmp_out2in_ed_slow_path (snat_main_t * sm, vlib_buffer_t * b0,
1613 ip4_header_t * ip0, icmp46_header_t * icmp0,
1614 u32 sw_if_index0, u32 rx_fib_index0,
1615 vlib_node_runtime_t * node, u32 next0, f64 now,
1616 u32 thread_index, snat_session_t ** p_s0)
1618 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1619 next0, thread_index, p_s0, 0);
1620 snat_session_t * s0 = *p_s0;
1621 if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0))
1624 nat44_session_update_counters (s0, now,
1625 vlib_buffer_length_in_chain (sm->vlib_main, b0));
1631 nat44_o2i_ed_is_idle_session_cb (clib_bihash_kv_16_8_t * kv, void * arg)
1633 snat_main_t *sm = &snat_main;
1634 nat44_is_idle_session_ctx_t *ctx = arg;
1636 u64 sess_timeout_time;
1637 nat_ed_ses_key_t ed_key;
1638 clib_bihash_kv_16_8_t ed_kv;
1641 snat_session_key_t key;
1642 snat_main_per_thread_data_t *tsm = vec_elt_at_index (sm->per_thread_data,
1645 s = pool_elt_at_index (tsm->sessions, kv->value);
1646 sess_timeout_time = s->last_heard + (f64)nat44_session_get_timeout(sm, s);
1647 if (ctx->now >= sess_timeout_time)
1649 ed_key.l_addr = s->in2out.addr;
1650 ed_key.r_addr = s->ext_host_addr;
1651 ed_key.fib_index = s->out2in.fib_index;
1652 if (snat_is_unk_proto_session (s))
1654 ed_key.proto = s->in2out.port;
1660 ed_key.proto = snat_proto_to_ip_proto (s->in2out.protocol);
1661 ed_key.l_port = s->in2out.port;
1662 ed_key.r_port = s->ext_host_port;
1664 if (is_twice_nat_session (s))
1666 ed_key.r_addr = s->ext_host_nat_addr;
1667 ed_key.r_port = s->ext_host_nat_port;
1669 ed_kv.key[0] = ed_key.as_u64[0];
1670 ed_kv.key[1] = ed_key.as_u64[1];
1671 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
1672 nat_log_warn ("in2out_ed key del failed");
1674 if (snat_is_unk_proto_session (s))
1677 snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
1678 s->out2in.addr.as_u32,
1682 s->in2out.fib_index);
1684 if (is_twice_nat_session (s))
1686 for (i = 0; i < vec_len (sm->twice_nat_addresses); i++)
1688 key.protocol = s->in2out.protocol;
1689 key.port = s->ext_host_nat_port;
1690 a = sm->twice_nat_addresses + i;
1691 if (a->addr.as_u32 == s->ext_host_nat_addr.as_u32)
1693 snat_free_outside_address_and_port (sm->twice_nat_addresses,
1694 ctx->thread_index, &key);
1700 if (snat_is_session_static (s))
1703 if (s->outside_address_index != ~0)
1704 snat_free_outside_address_and_port (sm->addresses, ctx->thread_index,
1707 nat44_delete_session (sm, s, ctx->thread_index);
1714 static snat_session_t *
1715 create_session_for_static_mapping_ed (snat_main_t * sm,
1717 snat_session_key_t l_key,
1718 snat_session_key_t e_key,
1719 vlib_node_runtime_t * node,
1721 twice_nat_type_t twice_nat,
1729 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1730 clib_bihash_kv_16_8_t kv;
1731 snat_session_key_t eh_key;
1733 nat44_is_idle_session_ctx_t ctx;
1735 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1737 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
1738 nat_log_notice ("maximum sessions exceeded");
1742 u = nat_user_get_or_create (sm, &l_key.addr, l_key.fib_index, thread_index);
1745 nat_log_warn ("create NAT user failed");
1749 s = nat_ed_session_alloc (sm, u, thread_index);
1752 nat44_delete_user_with_no_session (sm, u, thread_index);
1753 nat_log_warn ("create NAT session failed");
1757 ip = vlib_buffer_get_current (b);
1758 udp = ip4_next_header (ip);
1760 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
1761 s->ext_host_port = e_key.protocol == SNAT_PROTOCOL_ICMP ? 0 : udp->src_port;
1762 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1764 s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1765 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1766 s->outside_address_index = ~0;
1769 s->in2out.protocol = s->out2in.protocol;
1770 user_session_increment (sm, u, 1);
1772 /* Add to lookup tables */
1773 make_ed_kv (&kv, &e_key.addr, &s->ext_host_addr, ip->protocol,
1774 e_key.fib_index, e_key.port, s->ext_host_port);
1775 kv.value = s - tsm->sessions;
1777 ctx.thread_index = thread_index;
1778 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->out2in_ed, &kv,
1779 nat44_o2i_ed_is_idle_session_cb,
1781 nat_log_notice ("out2in-ed key add failed");
1783 if (twice_nat == TWICE_NAT || (twice_nat == TWICE_NAT_SELF &&
1784 ip->src_address.as_u32 == l_key.addr.as_u32))
1786 eh_key.protocol = e_key.protocol;
1787 if (snat_alloc_outside_address_and_port (sm->twice_nat_addresses, 0,
1788 thread_index, &eh_key,
1790 sm->port_per_thread,
1791 tsm->snat_thread_index))
1793 b->error = node->errors[SNAT_OUT2IN_ERROR_OUT_OF_PORTS];
1794 nat44_delete_session (sm, s, thread_index);
1795 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &kv, 0))
1796 nat_log_notice ("out2in-ed key del failed");
1799 s->ext_host_nat_addr.as_u32 = eh_key.addr.as_u32;
1800 s->ext_host_nat_port = eh_key.port;
1801 s->flags |= SNAT_SESSION_FLAG_TWICE_NAT;
1802 make_ed_kv (&kv, &l_key.addr, &s->ext_host_nat_addr, ip->protocol,
1803 l_key.fib_index, l_key.port, s->ext_host_nat_port);
1807 make_ed_kv (&kv, &l_key.addr, &s->ext_host_addr, ip->protocol,
1808 l_key.fib_index, l_key.port, s->ext_host_port);
1810 kv.value = s - tsm->sessions;
1811 if (clib_bihash_add_or_overwrite_stale_16_8 (&tsm->in2out_ed, &kv,
1812 nat44_i2o_ed_is_idle_session_cb,
1814 nat_log_notice ("in2out-ed key add failed");
1819 static_always_inline int
1820 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
1822 icmp46_header_t *icmp0;
1823 nat_ed_ses_key_t key0;
1824 icmp_echo_header_t *echo0, *inner_echo0 = 0;
1825 ip4_header_t *inner_ip0;
1826 void *l4_header = 0;
1827 icmp46_header_t *inner_icmp0;
1829 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
1830 echo0 = (icmp_echo_header_t *)(icmp0+1);
1832 if (!icmp_is_error_message (icmp0))
1834 key0.proto = IP_PROTOCOL_ICMP;
1835 key0.l_addr = ip0->dst_address;
1836 key0.r_addr = ip0->src_address;
1837 key0.l_port = echo0->identifier;
1842 inner_ip0 = (ip4_header_t *)(echo0+1);
1843 l4_header = ip4_next_header (inner_ip0);
1844 key0.proto = inner_ip0->protocol;
1845 key0.l_addr = inner_ip0->src_address;
1846 key0.r_addr = inner_ip0->dst_address;
1847 switch (ip_proto_to_snat_proto (inner_ip0->protocol))
1849 case SNAT_PROTOCOL_ICMP:
1850 inner_icmp0 = (icmp46_header_t*)l4_header;
1851 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
1852 key0.l_port = inner_echo0->identifier;
1855 case SNAT_PROTOCOL_UDP:
1856 case SNAT_PROTOCOL_TCP:
1857 key0.l_port = ((tcp_udp_header_t*)l4_header)->src_port;
1858 key0.r_port = ((tcp_udp_header_t*)l4_header)->dst_port;
1869 next_src_nat (snat_main_t * sm, ip4_header_t * ip, u8 proto, u16 src_port,
1870 u16 dst_port, u32 thread_index, u32 rx_fib_index)
1872 clib_bihash_kv_16_8_t kv, value;
1873 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1875 make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
1876 rx_fib_index, src_port, dst_port);
1877 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1884 create_bypass_for_fwd(snat_main_t * sm, ip4_header_t * ip, u32 rx_fib_index,
1887 nat_ed_ses_key_t key;
1888 clib_bihash_kv_16_8_t kv, value;
1891 snat_session_t *s = 0;
1892 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1893 f64 now = vlib_time_now (sm->vlib_main);
1895 if (ip->protocol == IP_PROTOCOL_ICMP)
1897 if (icmp_get_ed_key (ip, &key))
1900 else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
1902 udp = ip4_next_header(ip);
1903 key.r_addr = ip->src_address;
1904 key.l_addr = ip->dst_address;
1905 key.proto = ip->protocol;
1906 key.l_port = udp->dst_port;
1907 key.r_port = udp->src_port;
1911 key.r_addr = ip->src_address;
1912 key.l_addr = ip->dst_address;
1913 key.proto = ip->protocol;
1914 key.l_port = key.r_port = 0;
1917 kv.key[0] = key.as_u64[0];
1918 kv.key[1] = key.as_u64[1];
1920 if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
1922 s = pool_elt_at_index (tsm->sessions, value.value);
1926 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1929 u = nat_user_get_or_create (sm, &ip->dst_address, sm->inside_fib_index,
1933 nat_log_warn ("create NAT user failed");
1937 s = nat_ed_session_alloc (sm, u, thread_index);
1940 nat44_delete_user_with_no_session (sm, u, thread_index);
1941 nat_log_warn ("create NAT session failed");
1945 s->ext_host_addr = key.r_addr;
1946 s->ext_host_port = key.r_port;
1947 s->flags |= SNAT_SESSION_FLAG_FWD_BYPASS;
1948 s->outside_address_index = ~0;
1949 s->out2in.addr = key.l_addr;
1950 s->out2in.port = key.l_port;
1951 s->out2in.protocol = ip_proto_to_snat_proto (key.proto);
1952 s->out2in.fib_index = 0;
1953 s->in2out = s->out2in;
1954 user_session_increment (sm, u, 0);
1956 kv.value = s - tsm->sessions;
1957 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &kv, 1))
1958 nat_log_notice ("in2out_ed key add failed");
1961 if (ip->protocol == IP_PROTOCOL_TCP)
1963 tcp_header_t *tcp = ip4_next_header(ip);
1964 if (nat44_set_tcp_session_state_o2i (sm, s, tcp, thread_index))
1969 nat44_session_update_counters (s, now, 0);
1973 icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
1974 u32 thread_index, vlib_buffer_t * b, ip4_header_t * ip,
1975 u8 * p_proto, snat_session_key_t * p_value,
1976 u8 * p_dont_translate, void * d, void * e)
1978 u32 next = ~0, sw_if_index, rx_fib_index;
1979 icmp46_header_t *icmp;
1980 nat_ed_ses_key_t key;
1981 clib_bihash_kv_16_8_t kv, value;
1982 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1983 snat_session_t *s = 0;
1984 u8 dont_translate = 0, is_addr_only;
1985 snat_session_key_t e_key, l_key;
1987 icmp = (icmp46_header_t *) ip4_next_header (ip);
1988 sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
1989 rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
1991 if (icmp_get_ed_key (ip, &key))
1993 b->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
1994 next = SNAT_OUT2IN_NEXT_DROP;
1997 key.fib_index = rx_fib_index;
1998 kv.key[0] = key.as_u64[0];
1999 kv.key[1] = key.as_u64[1];
2001 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2003 /* Try to match static mapping */
2004 e_key.addr = ip->dst_address;
2005 e_key.port = key.l_port;
2006 e_key.protocol = ip_proto_to_snat_proto (key.proto);
2007 e_key.fib_index = rx_fib_index;
2008 if (snat_static_mapping_match(sm, e_key, &l_key, 1, &is_addr_only, 0, 0))
2010 if (!sm->forwarding_enabled)
2012 /* Don't NAT packet aimed at the intfc address */
2013 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index,
2014 ip->dst_address.as_u32)))
2019 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2020 next = NAT44_ED_OUT2IN_NEXT_DROP;
2026 if (next_src_nat(sm, ip, key.proto, key.l_port, key.r_port,
2027 thread_index, rx_fib_index))
2029 next = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2032 create_bypass_for_fwd(sm, ip, rx_fib_index, thread_index);
2037 if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
2038 (icmp->type != ICMP4_echo_request || !is_addr_only)))
2040 b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2041 next = NAT44_ED_OUT2IN_NEXT_DROP;
2045 /* Create session initiated by host from external network */
2046 s = create_session_for_static_mapping_ed(sm, b, l_key, e_key, node,
2048 vlib_time_now (sm->vlib_main));
2052 next = NAT44_ED_OUT2IN_NEXT_DROP;
2058 if (PREDICT_FALSE(icmp->type != ICMP4_echo_reply &&
2059 icmp->type != ICMP4_echo_request &&
2060 !icmp_is_error_message (icmp)))
2062 b->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
2063 next = SNAT_OUT2IN_NEXT_DROP;
2067 s = pool_elt_at_index (tsm->sessions, value.value);
2070 *p_proto = ip_proto_to_snat_proto (key.proto);
2073 *p_value = s->in2out;
2074 *p_dont_translate = dont_translate;
2076 *(snat_session_t**)d = s;
2080 static snat_session_t *
2081 nat44_ed_out2in_unknown_proto (snat_main_t *sm,
2088 vlib_node_runtime_t * node)
2090 clib_bihash_kv_8_8_t kv, value;
2091 clib_bihash_kv_16_8_t s_kv, s_value;
2092 snat_static_mapping_t *m;
2093 u32 old_addr, new_addr;
2096 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2099 old_addr = ip->dst_address.as_u32;
2101 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2102 rx_fib_index, 0, 0);
2104 if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2106 s = pool_elt_at_index (tsm->sessions, s_value.value);
2107 new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
2111 if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
2113 b->error = node->errors[SNAT_OUT2IN_ERROR_MAX_SESSIONS_EXCEEDED];
2114 nat_log_notice ("maximum sessions exceeded");
2118 make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
2119 if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2121 b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2125 m = pool_elt_at_index (sm->static_mappings, value.value);
2127 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
2129 u = nat_user_get_or_create (sm, &m->local_addr, m->fib_index,
2133 nat_log_warn ("create NAT user failed");
2137 /* Create a new session */
2138 s = nat_ed_session_alloc (sm, u, thread_index);
2141 nat44_delete_user_with_no_session (sm, u, thread_index);
2142 nat_log_warn ("create NAT session failed");
2146 s->ext_host_addr.as_u32 = ip->src_address.as_u32;
2147 s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
2148 s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2149 s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2150 s->outside_address_index = ~0;
2151 s->out2in.addr.as_u32 = old_addr;
2152 s->out2in.fib_index = rx_fib_index;
2153 s->in2out.addr.as_u32 = new_addr;
2154 s->in2out.fib_index = m->fib_index;
2155 s->in2out.port = s->out2in.port = ip->protocol;
2156 user_session_increment (sm, u, 1);
2158 /* Add to lookup tables */
2159 s_kv.value = s - tsm->sessions;
2160 if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2161 nat_log_notice ("out2in key add failed");
2163 make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2164 m->fib_index, 0, 0);
2165 s_kv.value = s - tsm->sessions;
2166 if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2167 nat_log_notice ("in2out key add failed");
2170 /* Update IP checksum */
2172 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2173 ip->checksum = ip_csum_fold (sum);
2175 vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2178 nat44_session_update_counters (s, now,
2179 vlib_buffer_length_in_chain (vm, b));
2185 nat44_ed_out2in_node_fn_inline (vlib_main_t * vm,
2186 vlib_node_runtime_t * node,
2187 vlib_frame_t * frame, int is_slow_path)
2189 u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2190 nat44_ed_out2in_next_t next_index;
2191 snat_main_t *sm = &snat_main;
2192 f64 now = vlib_time_now (vm);
2193 u32 thread_index = vm->thread_index;
2194 snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2196 stats_node_index = is_slow_path ? nat44_ed_out2in_slowpath_node.index :
2197 nat44_ed_out2in_node.index;
2199 from = vlib_frame_vector_args (frame);
2200 n_left_from = frame->n_vectors;
2201 next_index = node->cached_next_index;
2203 while (n_left_from > 0)
2207 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2209 while (n_left_from >= 4 && n_left_to_next >= 2)
2212 vlib_buffer_t *b0, *b1;
2213 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2214 u32 next1, sw_if_index1, rx_fib_index1, proto1, old_addr1, new_addr1;
2215 u16 old_port0, new_port0, old_port1, new_port1;
2216 ip4_header_t *ip0, *ip1;
2217 udp_header_t *udp0, *udp1;
2218 tcp_header_t *tcp0, *tcp1;
2219 icmp46_header_t *icmp0, *icmp1;
2220 snat_session_t *s0 = 0, *s1 = 0;
2221 clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
2222 ip_csum_t sum0, sum1;
2223 snat_session_key_t e_key0, l_key0, e_key1, l_key1;
2225 twice_nat_type_t twice_nat0, twice_nat1;
2227 /* Prefetch next iteration. */
2229 vlib_buffer_t * p2, * p3;
2231 p2 = vlib_get_buffer (vm, from[2]);
2232 p3 = vlib_get_buffer (vm, from[3]);
2234 vlib_prefetch_buffer_header (p2, LOAD);
2235 vlib_prefetch_buffer_header (p3, LOAD);
2237 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2238 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2241 /* speculatively enqueue b0 and b1 to the current next frame */
2242 to_next[0] = bi0 = from[0];
2243 to_next[1] = bi1 = from[1];
2247 n_left_to_next -= 2;
2249 b0 = vlib_get_buffer (vm, bi0);
2250 b1 = vlib_get_buffer (vm, bi1);
2252 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2253 vnet_buffer (b0)->snat.flags = 0;
2254 ip0 = vlib_buffer_get_current (b0);
2256 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2257 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2260 if (PREDICT_FALSE(ip0->ttl == 1))
2262 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2263 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2264 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2266 next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2270 udp0 = ip4_next_header (ip0);
2271 tcp0 = (tcp_header_t *) udp0;
2272 icmp0 = (icmp46_header_t *) udp0;
2273 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2277 if (PREDICT_FALSE (proto0 == ~0))
2279 s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2280 thread_index, now, vm, node);
2281 if (!sm->forwarding_enabled)
2284 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2289 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2291 next0 = icmp_out2in_ed_slow_path
2292 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2293 next0, now, thread_index, &s0);
2299 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2301 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2305 if (ip4_is_fragment (ip0))
2307 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2308 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2313 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2314 rx_fib_index0, udp0->dst_port, udp0->src_port);
2316 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2320 /* Try to match static mapping by external address and port,
2321 destination address and port in packet */
2322 e_key0.addr = ip0->dst_address;
2323 e_key0.port = udp0->dst_port;
2324 e_key0.protocol = proto0;
2325 e_key0.fib_index = rx_fib_index0;
2326 if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2327 &twice_nat0, &is_lb0))
2330 * Send DHCP packets to the ipv4 stack, or we won't
2331 * be able to use dhcp client on the outside interface
2333 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2334 && (udp0->dst_port ==
2335 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2337 vnet_feature_next (&next0, b0);
2341 if (!sm->forwarding_enabled)
2343 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2344 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2348 if (next_src_nat(sm, ip0, ip0->protocol,
2349 udp0->src_port, udp0->dst_port,
2350 thread_index, rx_fib_index0))
2352 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2355 create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2361 /* Create session initiated by host from external network */
2362 s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2370 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2376 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2382 s0 = pool_elt_at_index (tsm->sessions, value0.value);
2385 old_addr0 = ip0->dst_address.as_u32;
2386 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2387 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2389 sum0 = ip0->checksum;
2390 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2392 if (PREDICT_FALSE (is_twice_nat_session (s0)))
2393 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2394 s0->ext_host_nat_addr.as_u32, ip4_header_t,
2396 ip0->checksum = ip_csum_fold (sum0);
2398 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2400 old_port0 = tcp0->dst_port;
2401 new_port0 = tcp0->dst_port = s0->in2out.port;
2403 sum0 = tcp0->checksum;
2404 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2406 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2408 if (is_twice_nat_session (s0))
2410 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2411 s0->ext_host_nat_addr.as_u32,
2412 ip4_header_t, dst_address);
2413 sum0 = ip_csum_update (sum0, tcp0->src_port,
2414 s0->ext_host_nat_port, ip4_header_t,
2416 tcp0->src_port = s0->ext_host_nat_port;
2417 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2419 tcp0->checksum = ip_csum_fold(sum0);
2420 if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2425 udp0->dst_port = s0->in2out.port;
2426 if (is_twice_nat_session (s0))
2428 udp0->src_port = s0->ext_host_nat_port;
2429 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2435 nat44_session_update_counters (s0, now,
2436 vlib_buffer_length_in_chain (vm, b0));
2439 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2440 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2442 nat44_ed_out2in_trace_t *t =
2443 vlib_add_trace (vm, node, b0, sizeof (*t));
2444 t->is_slow_path = is_slow_path;
2445 t->sw_if_index = sw_if_index0;
2446 t->next_index = next0;
2447 t->session_index = ~0;
2449 t->session_index = s0 - tsm->sessions;
2452 pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2454 next1 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2455 vnet_buffer (b1)->snat.flags = 0;
2456 ip1 = vlib_buffer_get_current (b1);
2458 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2459 rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2462 if (PREDICT_FALSE(ip1->ttl == 1))
2464 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2465 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2466 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2468 next1 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2472 udp1 = ip4_next_header (ip1);
2473 tcp1 = (tcp_header_t *) udp1;
2474 icmp1 = (icmp46_header_t *) udp1;
2475 proto1 = ip_proto_to_snat_proto (ip1->protocol);
2479 if (PREDICT_FALSE (proto1 == ~0))
2481 s1 = nat44_ed_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
2482 thread_index, now, vm, node);
2483 if (!sm->forwarding_enabled)
2486 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2491 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
2493 next1 = icmp_out2in_ed_slow_path
2494 (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
2495 next1, now, thread_index, &s1);
2501 if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
2503 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2507 if (ip4_is_fragment (ip1))
2509 b1->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2510 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2515 make_ed_kv (&kv1, &ip1->dst_address, &ip1->src_address, ip1->protocol,
2516 rx_fib_index1, udp1->dst_port, udp1->src_port);
2518 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv1, &value1))
2522 /* Try to match static mapping by external address and port,
2523 destination address and port in packet */
2524 e_key1.addr = ip1->dst_address;
2525 e_key1.port = udp1->dst_port;
2526 e_key1.protocol = proto1;
2527 e_key1.fib_index = rx_fib_index1;
2528 if (snat_static_mapping_match(sm, e_key1, &l_key1, 1, 0,
2529 &twice_nat1, &is_lb1))
2532 * Send DHCP packets to the ipv4 stack, or we won't
2533 * be able to use dhcp client on the outside interface
2535 if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_UDP
2536 && (udp1->dst_port ==
2537 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2539 vnet_feature_next (&next1, b1);
2543 if (!sm->forwarding_enabled)
2545 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2546 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2550 if (next_src_nat(sm, ip1, ip1->protocol,
2551 udp1->src_port, udp1->dst_port,
2552 thread_index, rx_fib_index1))
2554 next1 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2557 create_bypass_for_fwd(sm, ip1, rx_fib_index1,
2563 /* Create session initiated by host from external network */
2564 s1 = create_session_for_static_mapping_ed(sm, b1, l_key1,
2572 next1 = NAT44_ED_OUT2IN_NEXT_DROP;
2578 next1 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2584 s1 = pool_elt_at_index (tsm->sessions, value1.value);
2587 old_addr1 = ip1->dst_address.as_u32;
2588 new_addr1 = ip1->dst_address.as_u32 = s1->in2out.addr.as_u32;
2589 vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->in2out.fib_index;
2591 sum1 = ip1->checksum;
2592 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2594 if (PREDICT_FALSE (is_twice_nat_session (s1)))
2595 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2596 s1->ext_host_nat_addr.as_u32, ip4_header_t,
2598 ip1->checksum = ip_csum_fold (sum1);
2600 if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
2602 old_port1 = tcp1->dst_port;
2603 new_port1 = tcp1->dst_port = s1->in2out.port;
2605 sum1 = tcp1->checksum;
2606 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
2608 sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
2610 if (is_twice_nat_session (s1))
2612 sum1 = ip_csum_update (sum1, ip1->src_address.as_u32,
2613 s1->ext_host_nat_addr.as_u32,
2614 ip4_header_t, dst_address);
2615 sum1 = ip_csum_update (sum1, tcp1->src_port,
2616 s1->ext_host_nat_port, ip4_header_t,
2618 tcp1->src_port = s1->ext_host_nat_port;
2619 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2621 tcp1->checksum = ip_csum_fold(sum1);
2622 if (nat44_set_tcp_session_state_o2i (sm, s1, tcp1, thread_index))
2627 udp1->dst_port = s1->in2out.port;
2628 if (is_twice_nat_session (s1))
2630 udp1->src_port = s1->ext_host_nat_port;
2631 ip1->src_address.as_u32 = s1->ext_host_nat_addr.as_u32;
2637 nat44_session_update_counters (s1, now,
2638 vlib_buffer_length_in_chain (vm, b1));
2641 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2642 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
2644 nat44_ed_out2in_trace_t *t =
2645 vlib_add_trace (vm, node, b1, sizeof (*t));
2646 t->is_slow_path = is_slow_path;
2647 t->sw_if_index = sw_if_index1;
2648 t->next_index = next1;
2649 t->session_index = ~0;
2651 t->session_index = s1 - tsm->sessions;
2654 pkts_processed += next1 != NAT44_ED_OUT2IN_NEXT_DROP;
2656 /* verify speculative enqueues, maybe switch current next frame */
2657 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2658 to_next, n_left_to_next,
2659 bi0, bi1, next0, next1);
2662 while (n_left_from > 0 && n_left_to_next > 0)
2666 u32 next0, sw_if_index0, rx_fib_index0, proto0, old_addr0, new_addr0;
2667 u16 old_port0, new_port0;
2671 icmp46_header_t * icmp0;
2672 snat_session_t *s0 = 0;
2673 clib_bihash_kv_16_8_t kv0, value0;
2675 snat_session_key_t e_key0, l_key0;
2677 twice_nat_type_t twice_nat0;
2679 /* speculatively enqueue b0 to the current next frame */
2685 n_left_to_next -= 1;
2687 b0 = vlib_get_buffer (vm, bi0);
2688 next0 = NAT44_ED_OUT2IN_NEXT_LOOKUP;
2689 vnet_buffer (b0)->snat.flags = 0;
2690 ip0 = vlib_buffer_get_current (b0);
2692 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2693 rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2696 if (PREDICT_FALSE(ip0->ttl == 1))
2698 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2699 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2700 ICMP4_time_exceeded_ttl_exceeded_in_transit,
2702 next0 = NAT44_ED_OUT2IN_NEXT_ICMP_ERROR;
2706 udp0 = ip4_next_header (ip0);
2707 tcp0 = (tcp_header_t *) udp0;
2708 icmp0 = (icmp46_header_t *) udp0;
2709 proto0 = ip_proto_to_snat_proto (ip0->protocol);
2713 if (PREDICT_FALSE (proto0 == ~0))
2715 s0 = nat44_ed_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
2716 thread_index, now, vm, node);
2717 if (!sm->forwarding_enabled)
2720 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2725 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2727 next0 = icmp_out2in_ed_slow_path
2728 (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2729 next0, now, thread_index, &s0);
2735 if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2737 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2741 if (ip4_is_fragment (ip0))
2743 b0->error = node->errors[SNAT_OUT2IN_ERROR_DROP_FRAGMENT];
2744 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2749 make_ed_kv (&kv0, &ip0->dst_address, &ip0->src_address, ip0->protocol,
2750 rx_fib_index0, udp0->dst_port, udp0->src_port);
2752 if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv0, &value0))
2756 /* Try to match static mapping by external address and port,
2757 destination address and port in packet */
2758 e_key0.addr = ip0->dst_address;
2759 e_key0.port = udp0->dst_port;
2760 e_key0.protocol = proto0;
2761 e_key0.fib_index = rx_fib_index0;
2762 if (snat_static_mapping_match(sm, e_key0, &l_key0, 1, 0,
2763 &twice_nat0, &is_lb0))
2766 * Send DHCP packets to the ipv4 stack, or we won't
2767 * be able to use dhcp client on the outside interface
2769 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_UDP
2770 && (udp0->dst_port ==
2771 clib_host_to_net_u16(UDP_DST_PORT_dhcp_to_client))))
2773 vnet_feature_next (&next0, b0);
2777 if (!sm->forwarding_enabled)
2779 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
2780 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2784 if (next_src_nat(sm, ip0, ip0->protocol,
2785 udp0->src_port, udp0->dst_port,
2786 thread_index, rx_fib_index0))
2788 next0 = NAT44_ED_OUT2IN_NEXT_IN2OUT;
2791 create_bypass_for_fwd(sm, ip0, rx_fib_index0,
2797 /* Create session initiated by host from external network */
2798 s0 = create_session_for_static_mapping_ed(sm, b0, l_key0,
2806 next0 = NAT44_ED_OUT2IN_NEXT_DROP;
2812 next0 = NAT44_ED_OUT2IN_NEXT_SLOW_PATH;
2818 s0 = pool_elt_at_index (tsm->sessions, value0.value);
2821 old_addr0 = ip0->dst_address.as_u32;
2822 new_addr0 = ip0->dst_address.as_u32 = s0->in2out.addr.as_u32;
2823 vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2825 sum0 = ip0->checksum;
2826 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2828 if (PREDICT_FALSE (is_twice_nat_session (s0)))
2829 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2830 s0->ext_host_nat_addr.as_u32, ip4_header_t,
2832 ip0->checksum = ip_csum_fold (sum0);
2834 if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
2836 old_port0 = tcp0->dst_port;
2837 new_port0 = tcp0->dst_port = s0->in2out.port;
2839 sum0 = tcp0->checksum;
2840 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
2842 sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
2844 if (is_twice_nat_session (s0))
2846 sum0 = ip_csum_update (sum0, ip0->src_address.as_u32,
2847 s0->ext_host_nat_addr.as_u32,
2848 ip4_header_t, dst_address);
2849 sum0 = ip_csum_update (sum0, tcp0->src_port,
2850 s0->ext_host_nat_port, ip4_header_t,
2852 tcp0->src_port = s0->ext_host_nat_port;
2853 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2855 tcp0->checksum = ip_csum_fold(sum0);
2856 if (nat44_set_tcp_session_state_o2i (sm, s0, tcp0, thread_index))
2861 udp0->dst_port = s0->in2out.port;
2862 if (is_twice_nat_session (s0))
2864 udp0->src_port = s0->ext_host_nat_port;
2865 ip0->src_address.as_u32 = s0->ext_host_nat_addr.as_u32;
2871 nat44_session_update_counters (s0, now,
2872 vlib_buffer_length_in_chain (vm, b0));
2875 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2876 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2878 nat44_ed_out2in_trace_t *t =
2879 vlib_add_trace (vm, node, b0, sizeof (*t));
2880 t->is_slow_path = is_slow_path;
2881 t->sw_if_index = sw_if_index0;
2882 t->next_index = next0;
2883 t->session_index = ~0;
2885 t->session_index = s0 - tsm->sessions;
2888 pkts_processed += next0 != NAT44_ED_OUT2IN_NEXT_DROP;
2890 /* verify speculative enqueue, maybe switch current next frame */
2891 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2892 to_next, n_left_to_next,
2896 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2899 vlib_node_increment_counter (vm, stats_node_index,
2900 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
2902 return frame->n_vectors;
2906 nat44_ed_out2in_fast_path_fn (vlib_main_t * vm,
2907 vlib_node_runtime_t * node,
2908 vlib_frame_t * frame)
2910 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 0);
2913 VLIB_REGISTER_NODE (nat44_ed_out2in_node) = {
2914 .function = nat44_ed_out2in_fast_path_fn,
2915 .name = "nat44-ed-out2in",
2916 .vector_size = sizeof (u32),
2917 .format_trace = format_nat44_ed_out2in_trace,
2918 .type = VLIB_NODE_TYPE_INTERNAL,
2920 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2921 .error_strings = snat_out2in_error_strings,
2923 .runtime_data_bytes = sizeof (snat_runtime_t),
2925 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2927 /* edit / add dispositions here */
2929 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2930 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2931 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2932 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2933 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2937 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_node, nat44_ed_out2in_fast_path_fn);
2940 nat44_ed_out2in_slow_path_fn (vlib_main_t * vm,
2941 vlib_node_runtime_t * node,
2942 vlib_frame_t * frame)
2944 return nat44_ed_out2in_node_fn_inline (vm, node, frame, 1);
2947 VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
2948 .function = nat44_ed_out2in_slow_path_fn,
2949 .name = "nat44-ed-out2in-slowpath",
2950 .vector_size = sizeof (u32),
2951 .format_trace = format_nat44_ed_out2in_trace,
2952 .type = VLIB_NODE_TYPE_INTERNAL,
2954 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
2955 .error_strings = snat_out2in_error_strings,
2957 .runtime_data_bytes = sizeof (snat_runtime_t),
2959 .n_next_nodes = NAT44_ED_OUT2IN_N_NEXT,
2961 /* edit / add dispositions here */
2963 [NAT44_ED_OUT2IN_NEXT_DROP] = "error-drop",
2964 [NAT44_ED_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
2965 [NAT44_ED_OUT2IN_NEXT_SLOW_PATH] = "nat44-ed-out2in-slowpath",
2966 [NAT44_ED_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2967 [NAT44_ED_OUT2IN_NEXT_IN2OUT] = "nat44-ed-in2out",
2971 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_out2in_slowpath_node,
2972 nat44_ed_out2in_slow_path_fn);
2974 /**************************/
2975 /*** deterministic mode ***/
2976 /**************************/
2978 snat_det_out2in_node_fn (vlib_main_t * vm,
2979 vlib_node_runtime_t * node,
2980 vlib_frame_t * frame)
2982 u32 n_left_from, * from, * to_next;
2983 snat_out2in_next_t next_index;
2984 u32 pkts_processed = 0;
2985 snat_main_t * sm = &snat_main;
2986 u32 thread_index = vm->thread_index;
2988 from = vlib_frame_vector_args (frame);
2989 n_left_from = frame->n_vectors;
2990 next_index = node->cached_next_index;
2992 while (n_left_from > 0)
2996 vlib_get_next_frame (vm, node, next_index,
2997 to_next, n_left_to_next);
2999 while (n_left_from >= 4 && n_left_to_next >= 2)
3002 vlib_buffer_t * b0, * b1;
3003 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
3004 u32 next1 = SNAT_OUT2IN_NEXT_LOOKUP;
3005 u32 sw_if_index0, sw_if_index1;
3006 ip4_header_t * ip0, * ip1;
3007 ip_csum_t sum0, sum1;
3008 ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
3009 u16 new_port0, old_port0, old_port1, new_port1;
3010 udp_header_t * udp0, * udp1;
3011 tcp_header_t * tcp0, * tcp1;
3013 snat_det_out_key_t key0, key1;
3014 snat_det_map_t * dm0, * dm1;
3015 snat_det_session_t * ses0 = 0, * ses1 = 0;
3016 u32 rx_fib_index0, rx_fib_index1;
3017 icmp46_header_t * icmp0, * icmp1;
3019 /* Prefetch next iteration. */
3021 vlib_buffer_t * p2, * p3;
3023 p2 = vlib_get_buffer (vm, from[2]);
3024 p3 = vlib_get_buffer (vm, from[3]);
3026 vlib_prefetch_buffer_header (p2, LOAD);
3027 vlib_prefetch_buffer_header (p3, LOAD);
3029 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3030 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3033 /* speculatively enqueue b0 and b1 to the current next frame */
3034 to_next[0] = bi0 = from[0];
3035 to_next[1] = bi1 = from[1];
3039 n_left_to_next -= 2;
3041 b0 = vlib_get_buffer (vm, bi0);
3042 b1 = vlib_get_buffer (vm, bi1);
3044 ip0 = vlib_buffer_get_current (b0);
3045 udp0 = ip4_next_header (ip0);
3046 tcp0 = (tcp_header_t *) udp0;
3048 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3050 if (PREDICT_FALSE(ip0->ttl == 1))
3052 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3053 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3054 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3056 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3060 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3062 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3064 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3065 icmp0 = (icmp46_header_t *) udp0;
3067 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3068 rx_fib_index0, node, next0, thread_index,
3073 key0.ext_host_addr = ip0->src_address;
3074 key0.ext_host_port = tcp0->src;
3075 key0.out_port = tcp0->dst;
3077 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
3078 if (PREDICT_FALSE(!dm0))
3080 nat_log_info ("unknown dst address: %U",
3081 format_ip4_address, &ip0->dst_address);
3082 next0 = SNAT_OUT2IN_NEXT_DROP;
3083 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3087 snat_det_reverse(dm0, &ip0->dst_address,
3088 clib_net_to_host_u16(tcp0->dst), &new_addr0);
3090 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3091 if (PREDICT_FALSE(!ses0))
3093 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3094 format_ip4_address, &ip0->src_address,
3095 clib_net_to_host_u16 (tcp0->src),
3096 format_ip4_address, &ip0->dst_address,
3097 clib_net_to_host_u16 (tcp0->dst),
3098 format_ip4_address, &new_addr0);
3099 next0 = SNAT_OUT2IN_NEXT_DROP;
3100 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3103 new_port0 = ses0->in_port;
3105 old_addr0 = ip0->dst_address;
3106 ip0->dst_address = new_addr0;
3107 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3109 sum0 = ip0->checksum;
3110 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3112 dst_address /* changed member */);
3113 ip0->checksum = ip_csum_fold (sum0);
3115 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3117 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3118 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3119 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
3120 snat_det_ses_close(dm0, ses0);
3122 old_port0 = tcp0->dst;
3123 tcp0->dst = new_port0;
3125 sum0 = tcp0->checksum;
3126 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3128 dst_address /* changed member */);
3130 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3131 ip4_header_t /* cheat */,
3132 length /* changed member */);
3133 tcp0->checksum = ip_csum_fold(sum0);
3137 old_port0 = udp0->dst_port;
3138 udp0->dst_port = new_port0;
3144 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3145 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3147 snat_out2in_trace_t *t =
3148 vlib_add_trace (vm, node, b0, sizeof (*t));
3149 t->sw_if_index = sw_if_index0;
3150 t->next_index = next0;
3151 t->session_index = ~0;
3153 t->session_index = ses0 - dm0->sessions;
3156 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3158 b1 = vlib_get_buffer (vm, bi1);
3160 ip1 = vlib_buffer_get_current (b1);
3161 udp1 = ip4_next_header (ip1);
3162 tcp1 = (tcp_header_t *) udp1;
3164 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3166 if (PREDICT_FALSE(ip1->ttl == 1))
3168 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3169 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3170 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3172 next1 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3176 proto1 = ip_proto_to_snat_proto (ip1->protocol);
3178 if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3180 rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3181 icmp1 = (icmp46_header_t *) udp1;
3183 next1 = icmp_out2in(sm, b1, ip1, icmp1, sw_if_index1,
3184 rx_fib_index1, node, next1, thread_index,
3189 key1.ext_host_addr = ip1->src_address;
3190 key1.ext_host_port = tcp1->src;
3191 key1.out_port = tcp1->dst;
3193 dm1 = snat_det_map_by_out(sm, &ip1->dst_address);
3194 if (PREDICT_FALSE(!dm1))
3196 nat_log_info ("unknown dst address: %U",
3197 format_ip4_address, &ip1->dst_address);
3198 next1 = SNAT_OUT2IN_NEXT_DROP;
3199 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3203 snat_det_reverse(dm1, &ip1->dst_address,
3204 clib_net_to_host_u16(tcp1->dst), &new_addr1);
3206 ses1 = snat_det_get_ses_by_out (dm1, &new_addr1, key1.as_u64);
3207 if (PREDICT_FALSE(!ses1))
3209 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3210 format_ip4_address, &ip1->src_address,
3211 clib_net_to_host_u16 (tcp1->src),
3212 format_ip4_address, &ip1->dst_address,
3213 clib_net_to_host_u16 (tcp1->dst),
3214 format_ip4_address, &new_addr1);
3215 next1 = SNAT_OUT2IN_NEXT_DROP;
3216 b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3219 new_port1 = ses1->in_port;
3221 old_addr1 = ip1->dst_address;
3222 ip1->dst_address = new_addr1;
3223 vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3225 sum1 = ip1->checksum;
3226 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3228 dst_address /* changed member */);
3229 ip1->checksum = ip_csum_fold (sum1);
3231 if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3233 if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3234 ses1->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3235 else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_LAST_ACK)
3236 snat_det_ses_close(dm1, ses1);
3238 old_port1 = tcp1->dst;
3239 tcp1->dst = new_port1;
3241 sum1 = tcp1->checksum;
3242 sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3244 dst_address /* changed member */);
3246 sum1 = ip_csum_update (sum1, old_port1, new_port1,
3247 ip4_header_t /* cheat */,
3248 length /* changed member */);
3249 tcp1->checksum = ip_csum_fold(sum1);
3253 old_port1 = udp1->dst_port;
3254 udp1->dst_port = new_port1;
3260 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3261 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3263 snat_out2in_trace_t *t =
3264 vlib_add_trace (vm, node, b1, sizeof (*t));
3265 t->sw_if_index = sw_if_index1;
3266 t->next_index = next1;
3267 t->session_index = ~0;
3269 t->session_index = ses1 - dm1->sessions;
3272 pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP;
3274 /* verify speculative enqueues, maybe switch current next frame */
3275 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3276 to_next, n_left_to_next,
3277 bi0, bi1, next0, next1);
3280 while (n_left_from > 0 && n_left_to_next > 0)
3284 u32 next0 = SNAT_OUT2IN_NEXT_LOOKUP;
3288 ip4_address_t new_addr0, old_addr0;
3289 u16 new_port0, old_port0;
3290 udp_header_t * udp0;
3291 tcp_header_t * tcp0;
3293 snat_det_out_key_t key0;
3294 snat_det_map_t * dm0;
3295 snat_det_session_t * ses0 = 0;
3297 icmp46_header_t * icmp0;
3299 /* speculatively enqueue b0 to the current next frame */
3305 n_left_to_next -= 1;
3307 b0 = vlib_get_buffer (vm, bi0);
3309 ip0 = vlib_buffer_get_current (b0);
3310 udp0 = ip4_next_header (ip0);
3311 tcp0 = (tcp_header_t *) udp0;
3313 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3315 if (PREDICT_FALSE(ip0->ttl == 1))
3317 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3318 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3319 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3321 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3325 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3327 if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3329 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3330 icmp0 = (icmp46_header_t *) udp0;
3332 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3333 rx_fib_index0, node, next0, thread_index,
3338 key0.ext_host_addr = ip0->src_address;
3339 key0.ext_host_port = tcp0->src;
3340 key0.out_port = tcp0->dst;
3342 dm0 = snat_det_map_by_out(sm, &ip0->dst_address);
3343 if (PREDICT_FALSE(!dm0))
3345 nat_log_info ("unknown dst address: %U",
3346 format_ip4_address, &ip0->dst_address);
3347 next0 = SNAT_OUT2IN_NEXT_DROP;
3348 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3352 snat_det_reverse(dm0, &ip0->dst_address,
3353 clib_net_to_host_u16(tcp0->dst), &new_addr0);
3355 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3356 if (PREDICT_FALSE(!ses0))
3358 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3359 format_ip4_address, &ip0->src_address,
3360 clib_net_to_host_u16 (tcp0->src),
3361 format_ip4_address, &ip0->dst_address,
3362 clib_net_to_host_u16 (tcp0->dst),
3363 format_ip4_address, &new_addr0);
3364 next0 = SNAT_OUT2IN_NEXT_DROP;
3365 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3368 new_port0 = ses0->in_port;
3370 old_addr0 = ip0->dst_address;
3371 ip0->dst_address = new_addr0;
3372 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->inside_fib_index;
3374 sum0 = ip0->checksum;
3375 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3377 dst_address /* changed member */);
3378 ip0->checksum = ip_csum_fold (sum0);
3380 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3382 if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3383 ses0->state = SNAT_SESSION_TCP_CLOSE_WAIT;
3384 else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_LAST_ACK)
3385 snat_det_ses_close(dm0, ses0);
3387 old_port0 = tcp0->dst;
3388 tcp0->dst = new_port0;
3390 sum0 = tcp0->checksum;
3391 sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3393 dst_address /* changed member */);
3395 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3396 ip4_header_t /* cheat */,
3397 length /* changed member */);
3398 tcp0->checksum = ip_csum_fold(sum0);
3402 old_port0 = udp0->dst_port;
3403 udp0->dst_port = new_port0;
3409 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3410 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3412 snat_out2in_trace_t *t =
3413 vlib_add_trace (vm, node, b0, sizeof (*t));
3414 t->sw_if_index = sw_if_index0;
3415 t->next_index = next0;
3416 t->session_index = ~0;
3418 t->session_index = ses0 - dm0->sessions;
3421 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3423 /* verify speculative enqueue, maybe switch current next frame */
3424 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3425 to_next, n_left_to_next,
3429 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3432 vlib_node_increment_counter (vm, snat_det_out2in_node.index,
3433 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3435 return frame->n_vectors;
3438 VLIB_REGISTER_NODE (snat_det_out2in_node) = {
3439 .function = snat_det_out2in_node_fn,
3440 .name = "nat44-det-out2in",
3441 .vector_size = sizeof (u32),
3442 .format_trace = format_snat_out2in_trace,
3443 .type = VLIB_NODE_TYPE_INTERNAL,
3445 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3446 .error_strings = snat_out2in_error_strings,
3448 .runtime_data_bytes = sizeof (snat_runtime_t),
3450 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3452 /* edit / add dispositions here */
3454 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3455 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3456 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3457 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3460 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_out2in_node, snat_det_out2in_node_fn);
3463 * Get address and port values to be used for ICMP packet translation
3464 * and create session if needed
3466 * @param[in,out] sm NAT main
3467 * @param[in,out] node NAT node runtime
3468 * @param[in] thread_index thread index
3469 * @param[in,out] b0 buffer containing packet to be translated
3470 * @param[out] p_proto protocol used for matching
3471 * @param[out] p_value address and port after NAT translation
3472 * @param[out] p_dont_translate if packet should not be translated
3473 * @param d optional parameter
3474 * @param e optional parameter
3476 u32 icmp_match_out2in_det(snat_main_t *sm, vlib_node_runtime_t *node,
3477 u32 thread_index, vlib_buffer_t *b0,
3478 ip4_header_t *ip0, u8 *p_proto,
3479 snat_session_key_t *p_value,
3480 u8 *p_dont_translate, void *d, void *e)
3482 icmp46_header_t *icmp0;
3485 snat_det_out_key_t key0;
3486 u8 dont_translate = 0;
3488 icmp_echo_header_t *echo0, *inner_echo0 = 0;
3489 ip4_header_t *inner_ip0;
3490 void *l4_header = 0;
3491 icmp46_header_t *inner_icmp0;
3492 snat_det_map_t * dm0 = 0;
3493 ip4_address_t new_addr0 = {{0}};
3494 snat_det_session_t * ses0 = 0;
3495 ip4_address_t out_addr;
3497 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3498 echo0 = (icmp_echo_header_t *)(icmp0+1);
3499 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3501 if (!icmp_is_error_message (icmp0))
3503 protocol = SNAT_PROTOCOL_ICMP;
3504 key0.ext_host_addr = ip0->src_address;
3505 key0.ext_host_port = 0;
3506 key0.out_port = echo0->identifier;
3507 out_addr = ip0->dst_address;
3511 inner_ip0 = (ip4_header_t *)(echo0+1);
3512 l4_header = ip4_next_header (inner_ip0);
3513 protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3514 key0.ext_host_addr = inner_ip0->dst_address;
3515 out_addr = inner_ip0->src_address;
3518 case SNAT_PROTOCOL_ICMP:
3519 inner_icmp0 = (icmp46_header_t*)l4_header;
3520 inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3521 key0.ext_host_port = 0;
3522 key0.out_port = inner_echo0->identifier;
3524 case SNAT_PROTOCOL_UDP:
3525 case SNAT_PROTOCOL_TCP:
3526 key0.ext_host_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3527 key0.out_port = ((tcp_udp_header_t*)l4_header)->src_port;
3530 b0->error = node->errors[SNAT_OUT2IN_ERROR_UNSUPPORTED_PROTOCOL];
3531 next0 = SNAT_OUT2IN_NEXT_DROP;
3536 dm0 = snat_det_map_by_out(sm, &out_addr);
3537 if (PREDICT_FALSE(!dm0))
3539 /* Don't NAT packet aimed at the intfc address */
3540 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3541 ip0->dst_address.as_u32)))
3546 nat_log_info ("unknown dst address: %U",
3547 format_ip4_address, &ip0->dst_address);
3551 snat_det_reverse(dm0, &ip0->dst_address,
3552 clib_net_to_host_u16(key0.out_port), &new_addr0);
3554 ses0 = snat_det_get_ses_by_out (dm0, &new_addr0, key0.as_u64);
3555 if (PREDICT_FALSE(!ses0))
3557 /* Don't NAT packet aimed at the intfc address */
3558 if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
3559 ip0->dst_address.as_u32)))
3564 nat_log_info ("no match src %U:%d dst %U:%d for user %U",
3565 format_ip4_address, &key0.ext_host_addr,
3566 clib_net_to_host_u16 (key0.ext_host_port),
3567 format_ip4_address, &out_addr,
3568 clib_net_to_host_u16 (key0.out_port),
3569 format_ip4_address, &new_addr0);
3570 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3571 next0 = SNAT_OUT2IN_NEXT_DROP;
3575 if (PREDICT_FALSE(icmp0->type != ICMP4_echo_reply &&
3576 !icmp_is_error_message (icmp0)))
3578 b0->error = node->errors[SNAT_OUT2IN_ERROR_BAD_ICMP_TYPE];
3579 next0 = SNAT_OUT2IN_NEXT_DROP;
3586 *p_proto = protocol;
3589 p_value->addr = new_addr0;
3590 p_value->fib_index = sm->inside_fib_index;
3591 p_value->port = ses0->in_port;
3593 *p_dont_translate = dont_translate;
3595 *(snat_det_session_t**)d = ses0;
3597 *(snat_det_map_t**)e = dm0;
3601 /**********************/
3602 /*** worker handoff ***/
3603 /**********************/
3605 snat_out2in_worker_handoff_fn (vlib_main_t * vm,
3606 vlib_node_runtime_t * node,
3607 vlib_frame_t * frame)
3609 snat_main_t *sm = &snat_main;
3610 vlib_thread_main_t *tm = vlib_get_thread_main ();
3611 u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
3612 static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3613 static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3615 vlib_frame_queue_elt_t *hf = 0;
3616 vlib_frame_queue_t *fq;
3617 vlib_frame_t *f = 0;
3619 u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3620 u32 next_worker_index = 0;
3621 u32 current_worker_index = ~0;
3622 u32 thread_index = vm->thread_index;
3623 vlib_frame_t *d = 0;
3625 ASSERT (vec_len (sm->workers));
3627 if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3629 vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3631 vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3632 tm->n_vlib_mains - 1,
3633 (vlib_frame_queue_t *) (~0));
3636 from = vlib_frame_vector_args (frame);
3637 n_left_from = frame->n_vectors;
3639 while (n_left_from > 0)
3652 b0 = vlib_get_buffer (vm, bi0);
3654 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3655 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3657 ip0 = vlib_buffer_get_current (b0);
3659 next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0);
3661 if (PREDICT_FALSE (next_worker_index != thread_index))
3665 if (next_worker_index != current_worker_index)
3667 fq = is_vlib_frame_queue_congested (
3668 sm->fq_out2in_index, next_worker_index, NAT_FQ_NELTS - 2,
3669 congested_handoff_queue_by_worker_index);
3673 /* if this is 1st frame */
3676 d = vlib_get_frame_to_node (vm, sm->error_node_index);
3677 to_next_drop = vlib_frame_vector_args (d);
3680 to_next_drop[0] = bi0;
3683 b0->error = node->errors[SNAT_OUT2IN_ERROR_FQ_CONGESTED];
3688 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3690 hf = vlib_get_worker_handoff_queue_elt (sm->fq_out2in_index,
3692 handoff_queue_elt_by_worker_index);
3694 n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3695 to_next_worker = &hf->buffer_index[hf->n_vectors];
3696 current_worker_index = next_worker_index;
3699 /* enqueue to correct worker thread */
3700 to_next_worker[0] = bi0;
3702 n_left_to_next_worker--;
3704 if (n_left_to_next_worker == 0)
3706 hf->n_vectors = VLIB_FRAME_SIZE;
3707 vlib_put_frame_queue_elt (hf);
3708 current_worker_index = ~0;
3709 handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3716 /* if this is 1st frame */
3719 f = vlib_get_frame_to_node (vm, sm->out2in_node_index);
3720 to_next = vlib_frame_vector_args (f);
3729 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3730 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3732 snat_out2in_worker_handoff_trace_t *t =
3733 vlib_add_trace (vm, node, b0, sizeof (*t));
3734 t->next_worker_index = next_worker_index;
3735 t->do_handoff = do_handoff;
3740 vlib_put_frame_to_node (vm, sm->out2in_node_index, f);
3743 vlib_put_frame_to_node (vm, sm->error_node_index, d);
3746 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3748 /* Ship frames to the worker nodes */
3749 for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3751 if (handoff_queue_elt_by_worker_index[i])
3753 hf = handoff_queue_elt_by_worker_index[i];
3755 * It works better to let the handoff node
3756 * rate-adapt, always ship the handoff queue element.
3758 if (1 || hf->n_vectors == hf->last_n_vectors)
3760 vlib_put_frame_queue_elt (hf);
3761 handoff_queue_elt_by_worker_index[i] = 0;
3764 hf->last_n_vectors = hf->n_vectors;
3766 congested_handoff_queue_by_worker_index[i] =
3767 (vlib_frame_queue_t *) (~0);
3770 current_worker_index = ~0;
3771 return frame->n_vectors;
3774 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
3775 .function = snat_out2in_worker_handoff_fn,
3776 .name = "nat44-out2in-worker-handoff",
3777 .vector_size = sizeof (u32),
3778 .format_trace = format_snat_out2in_worker_handoff_trace,
3779 .type = VLIB_NODE_TYPE_INTERNAL,
3781 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3782 .error_strings = snat_out2in_error_strings,
3791 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_worker_handoff_node, snat_out2in_worker_handoff_fn);
3794 snat_out2in_fast_node_fn (vlib_main_t * vm,
3795 vlib_node_runtime_t * node,
3796 vlib_frame_t * frame)
3798 u32 n_left_from, * from, * to_next;
3799 snat_out2in_next_t next_index;
3800 u32 pkts_processed = 0;
3801 snat_main_t * sm = &snat_main;
3803 from = vlib_frame_vector_args (frame);
3804 n_left_from = frame->n_vectors;
3805 next_index = node->cached_next_index;
3807 while (n_left_from > 0)
3811 vlib_get_next_frame (vm, node, next_index,
3812 to_next, n_left_to_next);
3814 while (n_left_from > 0 && n_left_to_next > 0)
3818 u32 next0 = SNAT_OUT2IN_NEXT_DROP;
3822 u32 new_addr0, old_addr0;
3823 u16 new_port0, old_port0;
3824 udp_header_t * udp0;
3825 tcp_header_t * tcp0;
3826 icmp46_header_t * icmp0;
3827 snat_session_key_t key0, sm0;
3831 /* speculatively enqueue b0 to the current next frame */
3837 n_left_to_next -= 1;
3839 b0 = vlib_get_buffer (vm, bi0);
3841 ip0 = vlib_buffer_get_current (b0);
3842 udp0 = ip4_next_header (ip0);
3843 tcp0 = (tcp_header_t *) udp0;
3844 icmp0 = (icmp46_header_t *) udp0;
3846 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3847 rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3849 vnet_feature_next (&next0, b0);
3851 if (PREDICT_FALSE(ip0->ttl == 1))
3853 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3854 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3855 ICMP4_time_exceeded_ttl_exceeded_in_transit,
3857 next0 = SNAT_OUT2IN_NEXT_ICMP_ERROR;
3861 proto0 = ip_proto_to_snat_proto (ip0->protocol);
3863 if (PREDICT_FALSE (proto0 == ~0))
3866 if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3868 next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0,
3869 rx_fib_index0, node, next0, ~0, 0, 0);
3873 key0.addr = ip0->dst_address;
3874 key0.port = udp0->dst_port;
3875 key0.fib_index = rx_fib_index0;
3877 if (snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
3879 b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
3883 new_addr0 = sm0.addr.as_u32;
3884 new_port0 = sm0.port;
3885 vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
3886 old_addr0 = ip0->dst_address.as_u32;
3887 ip0->dst_address.as_u32 = new_addr0;
3889 sum0 = ip0->checksum;
3890 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3892 dst_address /* changed member */);
3893 ip0->checksum = ip_csum_fold (sum0);
3895 if (PREDICT_FALSE(new_port0 != udp0->dst_port))
3897 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3899 old_port0 = tcp0->dst_port;
3900 tcp0->dst_port = new_port0;
3902 sum0 = tcp0->checksum;
3903 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3905 dst_address /* changed member */);
3907 sum0 = ip_csum_update (sum0, old_port0, new_port0,
3908 ip4_header_t /* cheat */,
3909 length /* changed member */);
3910 tcp0->checksum = ip_csum_fold(sum0);
3914 old_port0 = udp0->dst_port;
3915 udp0->dst_port = new_port0;
3921 if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3923 sum0 = tcp0->checksum;
3924 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
3926 dst_address /* changed member */);
3928 tcp0->checksum = ip_csum_fold(sum0);
3934 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3935 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3937 snat_out2in_trace_t *t =
3938 vlib_add_trace (vm, node, b0, sizeof (*t));
3939 t->sw_if_index = sw_if_index0;
3940 t->next_index = next0;
3943 pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP;
3945 /* verify speculative enqueue, maybe switch current next frame */
3946 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3947 to_next, n_left_to_next,
3951 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3954 vlib_node_increment_counter (vm, snat_out2in_fast_node.index,
3955 SNAT_OUT2IN_ERROR_OUT2IN_PACKETS,
3957 return frame->n_vectors;
3960 VLIB_REGISTER_NODE (snat_out2in_fast_node) = {
3961 .function = snat_out2in_fast_node_fn,
3962 .name = "nat44-out2in-fast",
3963 .vector_size = sizeof (u32),
3964 .format_trace = format_snat_out2in_fast_trace,
3965 .type = VLIB_NODE_TYPE_INTERNAL,
3967 .n_errors = ARRAY_LEN(snat_out2in_error_strings),
3968 .error_strings = snat_out2in_error_strings,
3970 .runtime_data_bytes = sizeof (snat_runtime_t),
3972 .n_next_nodes = SNAT_OUT2IN_N_NEXT,
3974 /* edit / add dispositions here */
3976 [SNAT_OUT2IN_NEXT_LOOKUP] = "ip4-lookup",
3977 [SNAT_OUT2IN_NEXT_DROP] = "error-drop",
3978 [SNAT_OUT2IN_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3979 [SNAT_OUT2IN_NEXT_REASS] = "nat44-out2in-reass",
3982 VLIB_NODE_FUNCTION_MULTIARCH (snat_out2in_fast_node, snat_out2in_fast_node_fn);