2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
17 * @brief NAT44 EI inside to outside network translation
20 #include <vlib/vlib.h>
22 #include <vnet/vnet.h>
23 #include <vnet/ip/ip.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/udp/udp_local.h>
26 #include <vnet/fib/ip4_fib.h>
28 #include <vppinfra/hash.h>
29 #include <vppinfra/error.h>
31 #include <nat/lib/log.h>
32 #include <nat/lib/nat_syslog.h>
33 #include <nat/lib/ipfix_logging.h>
34 #include <nat/lib/nat_inlines.h>
35 #include <nat/nat44-ei/nat44_ei_inlines.h>
36 #include <nat/nat44-ei/nat44_ei.h>
38 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
40 #define foreach_nat44_ei_in2out_error \
41 _ (UNSUPPORTED_PROTOCOL, "unsupported protocol") \
42 _ (OUT_OF_PORTS, "out of ports") \
43 _ (BAD_OUTSIDE_FIB, "outside VRF ID not found") \
44 _ (BAD_ICMP_TYPE, "unsupported ICMP type") \
45 _ (NO_TRANSLATION, "no translation") \
46 _ (MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded") \
47 _ (CANNOT_CREATE_USER, "cannot create NAT user")
49 #define foreach_nat44_ei_hairpinning_handoff_error \
50 _ (CONGESTION_DROP, "congestion drop")
54 #define _(sym, str) NAT44_EI_IN2OUT_ERROR_##sym,
55 foreach_nat44_ei_in2out_error
57 NAT44_EI_IN2OUT_N_ERROR,
58 } nat44_ei_in2out_error_t;
60 static char *nat44_ei_in2out_error_strings[] = {
61 #define _(sym,string) string,
62 foreach_nat44_ei_in2out_error
68 #define _(sym, str) NAT44_EI_HAIRPINNING_HANDOFF_ERROR_##sym,
69 foreach_nat44_ei_hairpinning_handoff_error
71 NAT44_EI_HAIRPINNING_HANDOFF_N_ERROR,
72 } nat44_ei_hairpinning_handoff_error_t;
74 static char *nat44_ei_hairpinning_handoff_error_strings[] = {
75 #define _(sym, string) string,
76 foreach_nat44_ei_hairpinning_handoff_error
82 NAT44_EI_IN2OUT_NEXT_LOOKUP,
83 NAT44_EI_IN2OUT_NEXT_DROP,
84 NAT44_EI_IN2OUT_NEXT_ICMP_ERROR,
85 NAT44_EI_IN2OUT_NEXT_SLOW_PATH,
86 NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF,
87 NAT44_EI_IN2OUT_N_NEXT,
88 } nat44_ei_in2out_next_t;
92 NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP,
93 NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP,
94 NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
95 } nat44_ei_in2out_hairpinnig_finish_next_t;
99 NAT44_EI_HAIRPIN_NEXT_LOOKUP,
100 NAT44_EI_HAIRPIN_NEXT_DROP,
101 NAT44_EI_HAIRPIN_NEXT_HANDOFF,
102 NAT44_EI_HAIRPIN_N_NEXT,
103 } nat44_ei_hairpin_next_t;
112 } nat44_ei_in2out_trace_t;
120 } nat44_ei_hairpin_trace_t;
124 u32 next_worker_index;
125 } nat44_ei_hairpinning_handoff_trace_t;
128 format_nat44_ei_in2out_trace (u8 *s, va_list *args)
130 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
131 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
132 nat44_ei_in2out_trace_t *t = va_arg (*args, nat44_ei_in2out_trace_t *);
134 tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
135 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
136 t->sw_if_index, t->next_index, t->session_index);
137 if (t->is_hairpinning)
138 s = format (s, ", with-hairpinning");
143 format_nat44_ei_in2out_fast_trace (u8 *s, va_list *args)
145 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
146 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
147 nat44_ei_in2out_trace_t *t = va_arg (*args, nat44_ei_in2out_trace_t *);
148 s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
149 t->sw_if_index, t->next_index);
154 format_nat44_ei_hairpin_trace (u8 *s, va_list *args)
156 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
157 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
158 nat44_ei_hairpin_trace_t *t = va_arg (*args, nat44_ei_hairpin_trace_t *);
160 s = format (s, "new dst addr %U port %u fib-index %u", format_ip4_address,
161 &t->addr, clib_net_to_host_u16 (t->port), t->fib_index);
162 if (~0 == t->session_index)
164 s = format (s, " is-static-mapping");
168 s = format (s, " session-index %u", t->session_index);
175 format_nat44_ei_hairpinning_handoff_trace (u8 *s, va_list *args)
177 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
178 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
179 nat44_ei_hairpinning_handoff_trace_t *t =
180 va_arg (*args, nat44_ei_hairpinning_handoff_trace_t *);
182 s = format (s, "nat44-ei-hairpinning-handoff: next-worker %d",
183 t->next_worker_index);
188 static_always_inline int
189 nat44_ei_not_translate_fast (vlib_node_runtime_t *node, u32 sw_if_index0,
190 ip4_header_t *ip0, u32 proto0, u32 rx_fib_index0)
192 nat44_ei_main_t *nm = &nat44_ei_main;
197 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
198 nat44_ei_outside_fib_t *outside_fib;
200 .fp_proto = FIB_PROTOCOL_IP4,
203 .ip4.as_u32 = ip0->dst_address.as_u32,
208 /* Don't NAT packet aimed at the intfc address */
209 if (PREDICT_FALSE (nat44_ei_is_interface_addr (
210 nm->ip4_main, node, sw_if_index0, ip0->dst_address.as_u32)))
213 fei = fib_table_lookup (rx_fib_index0, &pfx);
214 if (FIB_NODE_INDEX_INVALID != fei)
216 u32 sw_if_index = fib_entry_get_resolving_interface (fei);
217 if (sw_if_index == ~0)
219 vec_foreach (outside_fib, nm->outside_fibs)
221 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
222 if (FIB_NODE_INDEX_INVALID != fei)
224 sw_if_index = fib_entry_get_resolving_interface (fei);
225 if (sw_if_index != ~0)
230 if (sw_if_index == ~0)
233 nat44_ei_interface_t *i;
234 pool_foreach (i, nm->interfaces)
236 /* NAT packet aimed at outside interface */
237 if ((nat44_ei_interface_is_outside (i)) &&
238 (sw_if_index == i->sw_if_index))
246 static_always_inline int
247 nat44_ei_not_translate (nat44_ei_main_t *nm, vlib_node_runtime_t *node,
248 u32 sw_if_index0, ip4_header_t *ip0, u32 proto0,
249 u32 rx_fib_index0, u32 thread_index)
251 udp_header_t *udp0 = ip4_next_header (ip0);
252 clib_bihash_kv_8_8_t kv0, value0;
254 init_nat_k (&kv0, ip0->dst_address, udp0->dst_port, nm->outside_fib_index,
257 /* NAT packet aimed at external address if */
258 /* has active sessions */
259 if (clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
261 /* or is static mappings */
262 ip4_address_t placeholder_addr;
263 u16 placeholder_port;
264 u32 placeholder_fib_index;
265 if (!nat44_ei_static_mapping_match (ip0->dst_address, udp0->dst_port,
266 nm->outside_fib_index, proto0,
267 &placeholder_addr, &placeholder_port,
268 &placeholder_fib_index, 1, 0, 0))
274 if (nm->forwarding_enabled)
277 return nat44_ei_not_translate_fast (node, sw_if_index0, ip0, proto0,
281 static_always_inline int
282 nat44_ei_not_translate_output_feature (nat44_ei_main_t *nm, ip4_header_t *ip0,
283 u32 proto0, u16 src_port, u16 dst_port,
284 u32 thread_index, u32 sw_if_index)
286 clib_bihash_kv_8_8_t kv0, value0;
287 nat44_ei_interface_t *i;
290 init_nat_k (&kv0, ip0->src_address, src_port,
291 ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
293 if (!clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
297 init_nat_k (&kv0, ip0->dst_address, dst_port,
298 ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
299 if (!clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
302 pool_foreach (i, nm->output_feature_interfaces)
304 if ((nat44_ei_interface_is_inside (i)) &&
305 (sw_if_index == i->sw_if_index))
314 #ifndef CLIB_MARCH_VARIANT
316 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
318 nat44_ei_main_t *nm = &nat44_ei_main;
319 nat44_ei_is_idle_session_ctx_t *ctx = arg;
320 nat44_ei_session_t *s;
321 u64 sess_timeout_time;
322 nat44_ei_main_per_thread_data_t *tnm =
323 vec_elt_at_index (nm->per_thread_data, ctx->thread_index);
324 clib_bihash_kv_8_8_t s_kv;
326 if (ctx->thread_index != nat_value_get_thread_index (kv))
331 s = pool_elt_at_index (tnm->sessions, nat_value_get_session_index (kv));
332 sess_timeout_time = s->last_heard + (f64) nat_session_get_timeout (
333 &nm->timeouts, s->nat_proto, s->state);
334 if (ctx->now >= sess_timeout_time)
336 init_nat_o2i_k (&s_kv, s);
337 if (clib_bihash_add_del_8_8 (&nm->out2in, &s_kv, 0))
338 nat_elog_warn (nm, "out2in key del failed");
340 nat_ipfix_logging_nat44_ses_delete (
341 ctx->thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
342 nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
343 s->in2out.fib_index);
345 nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
346 &s->in2out.addr, s->in2out.port,
347 &s->out2in.addr, s->out2in.port, s->nat_proto);
349 nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
350 s->ext_host_port, s->nat_proto, s->out2in.fib_index,
353 if (!nat44_ei_is_session_static (s))
354 nat44_ei_free_outside_address_and_port (
355 nm->addresses, ctx->thread_index, &s->out2in.addr, s->out2in.port,
358 nat44_ei_delete_session (nm, s, ctx->thread_index);
367 slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0, ip4_header_t *ip0,
368 ip4_address_t i2o_addr, u16 i2o_port, u32 rx_fib_index0,
369 nat_protocol_t nat_proto, nat44_ei_session_t **sessionp,
370 vlib_node_runtime_t *node, u32 next0, u32 thread_index, f64 now)
373 nat44_ei_session_t *s = 0;
374 clib_bihash_kv_8_8_t kv0;
376 nat44_ei_outside_fib_t *outside_fib;
377 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
380 .fp_proto = FIB_PROTOCOL_IP4,
383 .ip4.as_u32 = ip0->dst_address.as_u32,
386 nat44_ei_is_idle_session_ctx_t ctx0;
387 ip4_address_t sm_addr;
391 if (PREDICT_FALSE (nat44_ei_maximum_sessions_exceeded (nm, thread_index)))
393 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
394 nat_ipfix_logging_max_sessions (thread_index,
395 nm->max_translations_per_thread);
396 nat_elog_notice (nm, "maximum sessions exceeded");
397 return NAT44_EI_IN2OUT_NEXT_DROP;
400 /* First try to match static mapping by local address and port */
401 if (nat44_ei_static_mapping_match (i2o_addr, i2o_port, rx_fib_index0,
402 nat_proto, &sm_addr, &sm_port,
403 &sm_fib_index, 0, 0, &identity_nat))
405 /* Try to create dynamic translation */
406 if (nm->alloc_addr_and_port (
407 nm->addresses, rx_fib_index0, thread_index, nat_proto,
408 ip0->src_address, &sm_addr, &sm_port, nm->port_per_thread,
409 nm->per_thread_data[thread_index].snat_thread_index))
411 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_OUT_OF_PORTS];
412 return NAT44_EI_IN2OUT_NEXT_DROP;
417 if (PREDICT_FALSE (identity_nat))
426 u = nat44_ei_user_get_or_create (nm, &ip0->src_address, rx_fib_index0,
430 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_CANNOT_CREATE_USER];
431 return NAT44_EI_IN2OUT_NEXT_DROP;
434 s = nat44_ei_session_alloc_or_recycle (nm, u, thread_index, now);
437 nat44_ei_delete_user_with_no_session (nm, u, thread_index);
438 nat_elog_warn (nm, "create NAT session failed");
439 return NAT44_EI_IN2OUT_NEXT_DROP;
443 s->flags |= NAT44_EI_SESSION_FLAG_STATIC_MAPPING;
444 nat44_ei_user_session_increment (nm, u, is_sm);
445 s->in2out.addr = i2o_addr;
446 s->in2out.port = i2o_port;
447 s->in2out.fib_index = rx_fib_index0;
448 s->nat_proto = nat_proto;
449 s->out2in.addr = sm_addr;
450 s->out2in.port = sm_port;
451 s->out2in.fib_index = nm->outside_fib_index;
452 switch (vec_len (nm->outside_fibs))
455 s->out2in.fib_index = nm->outside_fib_index;
458 s->out2in.fib_index = nm->outside_fibs[0].fib_index;
461 vec_foreach (outside_fib, nm->outside_fibs)
463 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
464 if (FIB_NODE_INDEX_INVALID != fei)
466 if (fib_entry_get_resolving_interface (fei) != ~0)
468 s->out2in.fib_index = outside_fib->fib_index;
475 s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
476 s->ext_host_port = vnet_buffer (b0)->ip.reass.l4_dst_port;
479 /* Add to translation hashes */
481 ctx0.thread_index = thread_index;
482 init_nat_i2o_kv (&kv0, s, thread_index,
483 s - nm->per_thread_data[thread_index].sessions);
484 if (clib_bihash_add_or_overwrite_stale_8_8 (
485 &nm->in2out, &kv0, nat44_i2o_is_idle_session_cb, &ctx0))
486 nat_elog_notice (nm, "in2out key add failed");
488 init_nat_o2i_kv (&kv0, s, thread_index,
489 s - nm->per_thread_data[thread_index].sessions);
490 if (clib_bihash_add_or_overwrite_stale_8_8 (
491 &nm->out2in, &kv0, nat44_o2i_is_idle_session_cb, &ctx0))
492 nat_elog_notice (nm, "out2in key add failed");
495 nat_ipfix_logging_nat44_ses_create (
496 thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
497 nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
498 s->in2out.fib_index);
500 nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index, &s->in2out.addr,
501 s->in2out.port, &s->out2in.addr, s->out2in.port,
504 nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
505 s->out2in.port, &s->ext_host_addr, s->ext_host_port,
506 &s->ext_host_nat_addr, s->ext_host_nat_port, s->nat_proto,
507 s->in2out.fib_index, s->flags, thread_index, 0);
512 static_always_inline nat44_ei_in2out_error_t
513 icmp_get_key (vlib_buffer_t *b, ip4_header_t *ip0, ip4_address_t *addr,
514 u16 *port, nat_protocol_t *nat_proto)
516 icmp46_header_t *icmp0;
517 icmp_echo_header_t *echo0, *inner_echo0 = 0;
518 ip4_header_t *inner_ip0 = 0;
520 icmp46_header_t *inner_icmp0;
522 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
523 echo0 = (icmp_echo_header_t *) (icmp0 + 1);
525 if (!icmp_type_is_error_message
526 (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
528 *nat_proto = NAT_PROTOCOL_ICMP;
529 *addr = ip0->src_address;
530 *port = vnet_buffer (b)->ip.reass.l4_src_port;
534 inner_ip0 = (ip4_header_t *) (echo0 + 1);
535 l4_header = ip4_next_header (inner_ip0);
536 *nat_proto = ip_proto_to_nat_proto (inner_ip0->protocol);
537 *addr = inner_ip0->dst_address;
540 case NAT_PROTOCOL_ICMP:
541 inner_icmp0 = (icmp46_header_t *) l4_header;
542 inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
543 *port = inner_echo0->identifier;
545 case NAT_PROTOCOL_UDP:
546 case NAT_PROTOCOL_TCP:
547 *port = ((tcp_udp_header_t *) l4_header)->dst_port;
550 return NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
553 return -1; /* success */
556 static_always_inline u32
557 nat44_ei_icmp_match_in2out_slow (vlib_node_runtime_t *node, u32 thread_index,
558 vlib_buffer_t *b0, ip4_header_t *ip0,
559 ip4_address_t *addr, u16 *port,
560 u32 *fib_index, nat_protocol_t *proto,
561 nat44_ei_session_t **p_s0, u8 *dont_translate)
563 nat44_ei_main_t *nm = &nat44_ei_main;
564 nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
566 nat44_ei_session_t *s0 = 0;
567 clib_bihash_kv_8_8_t kv0, value0;
570 vlib_main_t *vm = vlib_get_main ();
573 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
574 *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
576 err = icmp_get_key (b0, ip0, addr, port, proto);
579 b0->error = node->errors[err];
580 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
584 init_nat_k (&kv0, *addr, *port, *fib_index, *proto);
585 if (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
587 if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
589 if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
590 nm, ip0, *proto, *port, *port, thread_index, sw_if_index0)))
598 if (PREDICT_FALSE (nat44_ei_not_translate (
599 nm, node, sw_if_index0, ip0, NAT_PROTOCOL_ICMP, *fib_index,
608 (icmp_type_is_error_message
609 (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags)))
611 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
612 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
616 next0 = slow_path (nm, b0, ip0, *addr, *port, *fib_index, *proto, &s0,
617 node, next0, thread_index, vlib_time_now (vm));
619 if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
631 (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
633 && vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
635 && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
636 reass.icmp_type_or_tcp_flags)))
638 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
639 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
643 s0 = pool_elt_at_index (tnm->sessions,
644 nat_value_get_session_index (&value0));
650 *addr = s0->out2in.addr;
651 *port = s0->out2in.port;
652 *fib_index = s0->out2in.fib_index;
659 static_always_inline u32
660 nat44_ei_icmp_match_in2out_fast (vlib_node_runtime_t *node, u32 thread_index,
661 vlib_buffer_t *b0, ip4_header_t *ip0,
662 ip4_address_t *addr, u16 *port,
663 u32 *fib_index, nat_protocol_t *proto,
664 nat44_ei_session_t **s0, u8 *dont_translate)
672 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
673 *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
675 err = icmp_get_key (b0, ip0, addr, port, proto);
678 b0->error = node->errors[err];
679 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
683 ip4_address_t sm_addr;
687 if (nat44_ei_static_mapping_match (*addr, *port, *fib_index, *proto,
688 &sm_addr, &sm_port, &sm_fib_index, 0,
691 if (PREDICT_FALSE (nat44_ei_not_translate_fast (
692 node, sw_if_index0, ip0, IP_PROTOCOL_ICMP, *fib_index)))
698 if (icmp_type_is_error_message
699 (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
701 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
705 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_NO_TRANSLATION];
706 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
711 (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags != ICMP4_echo_request
712 && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
713 ICMP4_echo_reply || !is_addr_only)
714 && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
715 reass.icmp_type_or_tcp_flags)))
717 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
718 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
726 static_always_inline u32
727 nat44_ei_icmp_hairpinning (nat44_ei_main_t *nm, vlib_buffer_t *b0,
728 u32 thread_index, ip4_header_t *ip0,
729 icmp46_header_t *icmp0, u32 *required_thread_index)
731 clib_bihash_kv_8_8_t kv0, value0;
732 u32 old_dst_addr0, new_dst_addr0;
733 u32 old_addr0, new_addr0;
734 u16 old_port0, new_port0;
735 u16 old_checksum0, new_checksum0;
738 nat44_ei_session_t *s0;
739 nat44_ei_static_mapping_t *m0;
741 if (icmp_type_is_error_message (
742 vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
744 ip4_header_t *inner_ip0 = 0;
745 tcp_udp_header_t *l4_header = 0;
747 inner_ip0 = (ip4_header_t *) ((icmp_echo_header_t *) (icmp0 + 1) + 1);
748 l4_header = ip4_next_header (inner_ip0);
749 u32 protocol = ip_proto_to_nat_proto (inner_ip0->protocol);
751 if (protocol != NAT_PROTOCOL_TCP && protocol != NAT_PROTOCOL_UDP)
754 init_nat_k (&kv0, ip0->dst_address, l4_header->src_port,
755 nm->outside_fib_index, protocol);
756 if (clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
758 ti = nat_value_get_thread_index (&value0);
759 if (ti != thread_index)
761 *required_thread_index = ti;
764 si = nat_value_get_session_index (&value0);
765 s0 = pool_elt_at_index (nm->per_thread_data[ti].sessions, si);
766 new_dst_addr0 = s0->in2out.addr.as_u32;
767 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
769 /* update inner source IP address */
770 old_addr0 = inner_ip0->src_address.as_u32;
771 inner_ip0->src_address.as_u32 = new_dst_addr0;
772 new_addr0 = inner_ip0->src_address.as_u32;
773 sum0 = icmp0->checksum;
775 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, src_address);
776 icmp0->checksum = ip_csum_fold (sum0);
778 /* update inner IP header checksum */
779 old_checksum0 = inner_ip0->checksum;
780 sum0 = inner_ip0->checksum;
782 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, src_address);
783 inner_ip0->checksum = ip_csum_fold (sum0);
784 new_checksum0 = inner_ip0->checksum;
785 sum0 = icmp0->checksum;
786 sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
788 icmp0->checksum = ip_csum_fold (sum0);
790 /* update inner source port */
791 old_port0 = l4_header->src_port;
792 l4_header->src_port = s0->in2out.port;
793 new_port0 = l4_header->src_port;
794 sum0 = icmp0->checksum;
795 sum0 = ip_csum_update (sum0, old_port0, new_port0, tcp_udp_header_t,
797 icmp0->checksum = ip_csum_fold (sum0);
801 init_nat_k (&kv0, ip0->dst_address, 0, nm->outside_fib_index, 0);
802 if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv0,
805 icmp_echo_header_t *echo0 = (icmp_echo_header_t *) (icmp0 + 1);
806 u16 icmp_id0 = echo0->identifier;
807 init_nat_k (&kv0, ip0->dst_address, icmp_id0, nm->outside_fib_index,
809 int rv = clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0);
812 ti = nat_value_get_thread_index (&value0);
813 if (ti != thread_index)
815 *required_thread_index = ti;
818 si = nat_value_get_session_index (&value0);
819 s0 = pool_elt_at_index (nm->per_thread_data[ti].sessions, si);
820 new_dst_addr0 = s0->in2out.addr.as_u32;
821 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
822 echo0->identifier = s0->in2out.port;
823 sum0 = icmp0->checksum;
824 sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
825 icmp_echo_header_t, identifier);
826 icmp0->checksum = ip_csum_fold (sum0);
833 m0 = pool_elt_at_index (nm->static_mappings, value0.value);
835 new_dst_addr0 = m0->local_addr.as_u32;
836 if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
837 vnet_buffer (b0)->sw_if_index[VLIB_TX] = m0->fib_index;
840 /* Destination is behind the same NAT, use internal address and port */
843 old_dst_addr0 = ip0->dst_address.as_u32;
844 ip0->dst_address.as_u32 = new_dst_addr0;
845 sum0 = ip0->checksum;
846 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0, ip4_header_t,
848 ip0->checksum = ip_csum_fold (sum0);
853 static_always_inline u32
854 nat44_ei_icmp_in2out (vlib_buffer_t *b0, ip4_header_t *ip0,
855 icmp46_header_t *icmp0, u32 sw_if_index0,
856 u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0,
857 u32 thread_index, nat44_ei_session_t **p_s0)
859 nat44_ei_main_t *nm = &nat44_ei_main;
860 vlib_main_t *vm = vlib_get_main ();
864 nat_protocol_t proto;
865 icmp_echo_header_t *echo0, *inner_echo0 = 0;
866 ip4_header_t *inner_ip0;
868 icmp46_header_t *inner_icmp0;
870 u32 new_addr0, old_addr0;
871 u16 old_id0, new_id0;
872 u16 old_checksum0, new_checksum0;
876 u32 required_thread_index = thread_index;
878 echo0 = (icmp_echo_header_t *) (icmp0 + 1);
880 if (PREDICT_TRUE (nm->pat))
882 next0_tmp = nat44_ei_icmp_match_in2out_slow (
883 node, thread_index, b0, ip0, &addr, &port, &fib_index, &proto, p_s0,
888 next0_tmp = nat44_ei_icmp_match_in2out_fast (
889 node, thread_index, b0, ip0, &addr, &port, &fib_index, &proto, p_s0,
895 if (next0 == NAT44_EI_IN2OUT_NEXT_DROP || dont_translate)
898 if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
901 ip_incremental_checksum_buffer (vm, b0,
903 (u8 *) vlib_buffer_get_current (b0),
904 ntohs (ip0->length) -
905 ip4_header_bytes (ip0), 0);
906 checksum0 = ~ip_csum_fold (sum0);
907 if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
909 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
914 old_addr0 = ip0->src_address.as_u32;
915 new_addr0 = ip0->src_address.as_u32 = addr.as_u32;
917 sum0 = ip0->checksum;
918 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
919 src_address /* changed member */ );
920 ip0->checksum = ip_csum_fold (sum0);
922 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
924 if (icmp0->checksum == 0)
925 icmp0->checksum = 0xffff;
927 if (!icmp_type_is_error_message (icmp0->type))
930 if (PREDICT_FALSE (new_id0 != echo0->identifier))
932 old_id0 = echo0->identifier;
934 echo0->identifier = new_id0;
936 sum0 = icmp0->checksum;
938 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
940 icmp0->checksum = ip_csum_fold (sum0);
945 inner_ip0 = (ip4_header_t *) (echo0 + 1);
946 l4_header = ip4_next_header (inner_ip0);
948 if (!ip4_header_checksum_is_valid (inner_ip0))
950 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
954 /* update inner destination IP address */
955 old_addr0 = inner_ip0->dst_address.as_u32;
956 inner_ip0->dst_address = addr;
957 new_addr0 = inner_ip0->dst_address.as_u32;
958 sum0 = icmp0->checksum;
959 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
960 dst_address /* changed member */ );
961 icmp0->checksum = ip_csum_fold (sum0);
963 /* update inner IP header checksum */
964 old_checksum0 = inner_ip0->checksum;
965 sum0 = inner_ip0->checksum;
966 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
967 dst_address /* changed member */ );
968 inner_ip0->checksum = ip_csum_fold (sum0);
969 new_checksum0 = inner_ip0->checksum;
970 sum0 = icmp0->checksum;
972 ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
974 icmp0->checksum = ip_csum_fold (sum0);
978 case NAT_PROTOCOL_ICMP:
979 inner_icmp0 = (icmp46_header_t *) l4_header;
980 inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
982 old_id0 = inner_echo0->identifier;
984 inner_echo0->identifier = new_id0;
986 sum0 = icmp0->checksum;
988 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
990 icmp0->checksum = ip_csum_fold (sum0);
992 case NAT_PROTOCOL_UDP:
993 case NAT_PROTOCOL_TCP:
994 old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
996 ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
998 sum0 = icmp0->checksum;
999 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
1001 icmp0->checksum = ip_csum_fold (sum0);
1009 if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
1011 if (0 != nat44_ei_icmp_hairpinning (nm, b0, thread_index, ip0, icmp0,
1012 &required_thread_index))
1013 vnet_buffer (b0)->sw_if_index[VLIB_TX] = fib_index;
1014 if (thread_index != required_thread_index)
1016 vnet_buffer (b0)->snat.required_thread_index = required_thread_index;
1017 next0 = NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF;
1025 static_always_inline u32
1026 nat44_ei_icmp_in2out_slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0,
1027 ip4_header_t *ip0, icmp46_header_t *icmp0,
1028 u32 sw_if_index0, u32 rx_fib_index0,
1029 vlib_node_runtime_t *node, u32 next0, f64 now,
1030 u32 thread_index, nat44_ei_session_t **p_s0)
1032 vlib_main_t *vm = vlib_get_main ();
1034 next0 = nat44_ei_icmp_in2out (b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1035 node, next0, thread_index, p_s0);
1036 nat44_ei_session_t *s0 = *p_s0;
1037 if (PREDICT_TRUE (next0 != NAT44_EI_IN2OUT_NEXT_DROP && s0))
1040 nat44_ei_session_update_counters (
1041 s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1042 /* Per-user LRU list maintenance */
1043 nat44_ei_session_update_lru (nm, s0, thread_index);
1048 static_always_inline void
1049 nat44_ei_hairpinning_sm_unknown_proto (nat44_ei_main_t *nm, vlib_buffer_t *b,
1052 clib_bihash_kv_8_8_t kv, value;
1053 nat44_ei_static_mapping_t *m;
1054 u32 old_addr, new_addr;
1057 init_nat_k (&kv, ip->dst_address, 0, 0, 0);
1058 if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
1061 m = pool_elt_at_index (nm->static_mappings, value.value);
1063 old_addr = ip->dst_address.as_u32;
1064 new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1066 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1067 ip->checksum = ip_csum_fold (sum);
1069 if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
1070 vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
1074 nat_in2out_sm_unknown_proto (nat44_ei_main_t *nm, vlib_buffer_t *b,
1075 ip4_header_t *ip, u32 rx_fib_index)
1077 clib_bihash_kv_8_8_t kv, value;
1078 nat44_ei_static_mapping_t *m;
1079 u32 old_addr, new_addr;
1082 init_nat_k (&kv, ip->src_address, 0, rx_fib_index, 0);
1083 if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
1086 m = pool_elt_at_index (nm->static_mappings, value.value);
1088 old_addr = ip->src_address.as_u32;
1089 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1091 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1092 ip->checksum = ip_csum_fold (sum);
1096 if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
1098 vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
1099 nat44_ei_hairpinning_sm_unknown_proto (nm, b, ip);
1105 static_always_inline int
1106 nat44_ei_hairpinning (vlib_main_t *vm, vlib_node_runtime_t *node,
1107 nat44_ei_main_t *nm, u32 thread_index, vlib_buffer_t *b0,
1108 ip4_header_t *ip0, udp_header_t *udp0,
1109 tcp_header_t *tcp0, u32 proto0, int do_trace,
1110 u32 *required_thread_index)
1112 nat44_ei_session_t *s0 = NULL;
1113 clib_bihash_kv_8_8_t kv0, value0;
1115 u32 new_dst_addr0 = 0, old_dst_addr0, si = ~0;
1116 u16 new_dst_port0 = ~0, old_dst_port0;
1118 ip4_address_t sm0_addr;
1121 u32 old_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1123 /* Check if destination is static mappings */
1124 if (!nat44_ei_static_mapping_match (
1125 ip0->dst_address, udp0->dst_port, nm->outside_fib_index, proto0,
1126 &sm0_addr, &sm0_port, &sm0_fib_index, 1 /* by external */, 0, 0))
1128 new_dst_addr0 = sm0_addr.as_u32;
1129 new_dst_port0 = sm0_port;
1130 vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0_fib_index;
1132 /* or active session */
1135 init_nat_k (&kv0, ip0->dst_address, udp0->dst_port,
1136 nm->outside_fib_index, proto0);
1137 rv = clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0);
1144 if (thread_index != nat_value_get_thread_index (&value0))
1146 *required_thread_index = nat_value_get_thread_index (&value0);
1150 si = nat_value_get_session_index (&value0);
1151 s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions, si);
1152 new_dst_addr0 = s0->in2out.addr.as_u32;
1153 new_dst_port0 = s0->in2out.port;
1154 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1157 /* Check if anything has changed and if not, then return 0. This
1158 helps avoid infinite loop, repeating the three nodes
1159 nat44-hairpinning-->ip4-lookup-->ip4-local, in case nothing has
1161 old_dst_addr0 = ip0->dst_address.as_u32;
1162 old_dst_port0 = tcp0->dst;
1163 if (new_dst_addr0 == old_dst_addr0 && new_dst_port0 == old_dst_port0 &&
1164 vnet_buffer (b0)->sw_if_index[VLIB_TX] == old_sw_if_index)
1167 /* Destination is behind the same NAT, use internal address and port */
1170 old_dst_addr0 = ip0->dst_address.as_u32;
1171 ip0->dst_address.as_u32 = new_dst_addr0;
1172 sum0 = ip0->checksum;
1173 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0, ip4_header_t,
1175 ip0->checksum = ip_csum_fold (sum0);
1177 old_dst_port0 = tcp0->dst;
1178 if (PREDICT_TRUE (new_dst_port0 != old_dst_port0))
1180 if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1182 tcp0->dst = new_dst_port0;
1183 sum0 = tcp0->checksum;
1184 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1185 ip4_header_t, dst_address);
1186 sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
1187 ip4_header_t /* cheat */, length);
1188 tcp0->checksum = ip_csum_fold (sum0);
1192 udp0->dst_port = new_dst_port0;
1198 if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1200 sum0 = tcp0->checksum;
1201 sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1202 ip4_header_t, dst_address);
1203 tcp0->checksum = ip_csum_fold (sum0);
1211 if (do_trace && PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
1212 (b0->flags & VLIB_BUFFER_IS_TRACED)))
1214 nat44_ei_hairpin_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
1215 t->addr.as_u32 = new_dst_addr0;
1216 t->port = new_dst_port0;
1217 t->fib_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1220 t->session_index = si;
1224 t->session_index = ~0;
1230 static_always_inline uword
1231 nat44_ei_hairpinning_handoff_fn_inline (vlib_main_t *vm,
1232 vlib_node_runtime_t *node,
1233 vlib_frame_t *frame, u32 fq_index)
1235 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1236 u32 n_enq, n_left_from, *from;
1237 u16 thread_indices[VLIB_FRAME_SIZE], *ti;
1239 from = vlib_frame_vector_args (frame);
1240 n_left_from = frame->n_vectors;
1241 vlib_get_buffers (vm, from, bufs, n_left_from);
1244 ti = thread_indices;
1246 while (n_left_from > 0)
1248 ti[0] = vnet_buffer (b[0])->snat.required_thread_index;
1250 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
1251 (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1253 nat44_ei_hairpinning_handoff_trace_t *t =
1254 vlib_add_trace (vm, node, b[0], sizeof (*t));
1255 t->next_worker_index = ti[0];
1262 n_enq = vlib_buffer_enqueue_to_thread (vm, node, fq_index, from,
1263 thread_indices, frame->n_vectors, 1);
1265 if (n_enq < frame->n_vectors)
1266 vlib_node_increment_counter (
1267 vm, node->node_index, NAT44_EI_HAIRPINNING_HANDOFF_ERROR_CONGESTION_DROP,
1268 frame->n_vectors - n_enq);
1269 return frame->n_vectors;
1272 static_always_inline uword
1273 nat44_ei_in2out_node_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
1274 vlib_frame_t *frame, int is_slow_path,
1275 int is_output_feature)
1277 u32 n_left_from, *from;
1278 nat44_ei_main_t *nm = &nat44_ei_main;
1279 f64 now = vlib_time_now (vm);
1280 u32 thread_index = vm->thread_index;
1282 from = vlib_frame_vector_args (frame);
1283 n_left_from = frame->n_vectors;
1285 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1286 u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1287 vlib_get_buffers (vm, from, b, n_left_from);
1289 while (n_left_from >= 2)
1291 vlib_buffer_t *b0, *b1;
1293 u32 rx_sw_if_index0, rx_sw_if_index1;
1294 u32 tx_sw_if_index0, tx_sw_if_index1;
1295 u32 cntr_sw_if_index0, cntr_sw_if_index1;
1296 ip4_header_t *ip0, *ip1;
1297 ip_csum_t sum0, sum1;
1298 u32 new_addr0, old_addr0, new_addr1, old_addr1;
1299 u16 old_port0, new_port0, old_port1, new_port1;
1300 udp_header_t *udp0, *udp1;
1301 tcp_header_t *tcp0, *tcp1;
1302 icmp46_header_t *icmp0, *icmp1;
1303 u32 rx_fib_index0, rx_fib_index1;
1305 nat44_ei_session_t *s0 = 0, *s1 = 0;
1306 clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1307 u32 iph_offset0 = 0, iph_offset1 = 0;
1314 /* Prefetch next iteration. */
1315 if (PREDICT_TRUE (n_left_from >= 4))
1317 vlib_buffer_t *p2, *p3;
1322 vlib_prefetch_buffer_header (p2, LOAD);
1323 vlib_prefetch_buffer_header (p3, LOAD);
1325 clib_prefetch_load (p2->data);
1326 clib_prefetch_load (p3->data);
1329 if (is_output_feature)
1330 iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1332 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1335 udp0 = ip4_next_header (ip0);
1336 tcp0 = (tcp_header_t *) udp0;
1337 icmp0 = (icmp46_header_t *) udp0;
1339 rx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1340 tx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1342 is_output_feature ? tx_sw_if_index0 : rx_sw_if_index0;
1344 vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index0);
1346 next0 = next1 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
1348 if (PREDICT_FALSE (ip0->ttl == 1))
1350 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1351 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1352 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1354 next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1358 proto0 = ip_proto_to_nat_proto (ip0->protocol);
1360 /* Next configured feature, probably ip4-lookup */
1363 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1365 if (nat_in2out_sm_unknown_proto (nm, b0, ip0, rx_fib_index0))
1367 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1369 node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1371 vlib_increment_simple_counter (
1372 is_slow_path ? &nm->counters.slowpath.in2out.other :
1373 &nm->counters.fastpath.in2out.other,
1374 thread_index, cntr_sw_if_index0, 1);
1378 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1380 next0 = nat44_ei_icmp_in2out_slow_path (
1381 nm, b0, ip0, icmp0, rx_sw_if_index0, rx_fib_index0, node,
1382 next0, now, thread_index, &s0);
1383 vlib_increment_simple_counter (
1384 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1385 &nm->counters.fastpath.in2out.icmp,
1386 thread_index, cntr_sw_if_index0, 1);
1392 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1394 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1398 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1400 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1405 init_nat_k (&kv0, ip0->src_address,
1406 vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1408 if (PREDICT_FALSE (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0) !=
1413 if (is_output_feature)
1415 if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1417 vnet_buffer (b0)->ip.reass.l4_src_port,
1418 vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1423 * Send DHCP packets to the ipv4 stack, or we won't
1424 * be able to use dhcp client on the outside interface
1427 (proto0 == NAT_PROTOCOL_UDP
1428 && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1429 clib_host_to_net_u16
1430 (UDP_DST_PORT_dhcp_to_server))
1431 && ip0->dst_address.as_u32 == 0xffffffff))
1436 if (PREDICT_FALSE (nat44_ei_not_translate (
1437 nm, node, rx_sw_if_index0, ip0, proto0, rx_fib_index0,
1442 next0 = slow_path (nm, b0, ip0, ip0->src_address,
1443 vnet_buffer (b0)->ip.reass.l4_src_port,
1444 rx_fib_index0, proto0, &s0, node, next0,
1446 if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
1449 if (PREDICT_FALSE (!s0))
1454 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1459 s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1460 nat_value_get_session_index (&value0));
1462 b0->flags |= VNET_BUFFER_F_IS_NATED;
1464 old_addr0 = ip0->src_address.as_u32;
1465 ip0->src_address = s0->out2in.addr;
1466 new_addr0 = ip0->src_address.as_u32;
1467 if (!is_output_feature)
1468 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1470 sum0 = ip0->checksum;
1471 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1472 ip4_header_t, src_address /* changed member */ );
1473 ip0->checksum = ip_csum_fold (sum0);
1476 if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1478 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1480 old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1481 new_port0 = udp0->src_port = s0->out2in.port;
1482 sum0 = tcp0->checksum;
1483 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1485 dst_address /* changed member */ );
1486 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1487 ip4_header_t /* cheat */ ,
1488 length /* changed member */ );
1489 mss_clamping (nm->mss_clamping, tcp0, &sum0);
1490 tcp0->checksum = ip_csum_fold (sum0);
1492 vlib_increment_simple_counter (is_slow_path ?
1493 &nm->counters.slowpath.in2out.tcp :
1494 &nm->counters.fastpath.in2out.tcp,
1495 thread_index, cntr_sw_if_index0, 1);
1499 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1501 udp0->src_port = s0->out2in.port;
1502 if (PREDICT_FALSE (udp0->checksum))
1504 old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1505 new_port0 = udp0->src_port;
1506 sum0 = udp0->checksum;
1507 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address /* changed member */
1510 ip_csum_update (sum0, old_port0, new_port0,
1511 ip4_header_t /* cheat */ ,
1512 length /* changed member */ );
1513 udp0->checksum = ip_csum_fold (sum0);
1516 vlib_increment_simple_counter (is_slow_path ?
1517 &nm->counters.slowpath.in2out.udp :
1518 &nm->counters.fastpath.in2out.udp,
1519 thread_index, cntr_sw_if_index0, 1);
1523 nat44_ei_session_update_counters (
1524 s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1525 /* Per-user LRU list maintenance */
1526 nat44_ei_session_update_lru (nm, s0, thread_index);
1529 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1530 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1532 nat44_ei_in2out_trace_t *t =
1533 vlib_add_trace (vm, node, b0, sizeof (*t));
1534 t->is_slow_path = is_slow_path;
1535 t->sw_if_index = rx_sw_if_index0;
1536 t->next_index = next0;
1537 t->session_index = ~0;
1539 t->session_index = s0 - nm->per_thread_data[thread_index].sessions;
1542 if (next0 == NAT44_EI_IN2OUT_NEXT_DROP)
1544 vlib_increment_simple_counter (
1545 is_slow_path ? &nm->counters.slowpath.in2out.drops :
1546 &nm->counters.fastpath.in2out.drops,
1547 thread_index, cntr_sw_if_index0, 1);
1550 if (is_output_feature)
1551 iph_offset1 = vnet_buffer (b1)->ip.reass.save_rewrite_length;
1553 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1556 udp1 = ip4_next_header (ip1);
1557 tcp1 = (tcp_header_t *) udp1;
1558 icmp1 = (icmp46_header_t *) udp1;
1560 rx_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1561 tx_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_TX];
1563 is_output_feature ? tx_sw_if_index1 : rx_sw_if_index1;
1565 vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index1);
1567 if (PREDICT_FALSE (ip1->ttl == 1))
1569 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1570 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1571 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1573 next1 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1577 proto1 = ip_proto_to_nat_proto (ip1->protocol);
1579 /* Next configured feature, probably ip4-lookup */
1582 if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1584 if (nat_in2out_sm_unknown_proto (nm, b1, ip1, rx_fib_index1))
1586 next1 = NAT44_EI_IN2OUT_NEXT_DROP;
1588 node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1590 vlib_increment_simple_counter (
1591 is_slow_path ? &nm->counters.slowpath.in2out.other :
1592 &nm->counters.fastpath.in2out.other,
1593 thread_index, cntr_sw_if_index1, 1);
1597 if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1599 next1 = nat44_ei_icmp_in2out_slow_path (
1600 nm, b1, ip1, icmp1, rx_sw_if_index1, rx_fib_index1, node,
1601 next1, now, thread_index, &s1);
1602 vlib_increment_simple_counter (
1603 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1604 &nm->counters.fastpath.in2out.icmp,
1605 thread_index, cntr_sw_if_index1, 1);
1611 if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1613 next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1617 if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1619 next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1624 init_nat_k (&kv1, ip1->src_address,
1625 vnet_buffer (b1)->ip.reass.l4_src_port, rx_fib_index1,
1627 if (PREDICT_FALSE (clib_bihash_search_8_8 (&nm->in2out, &kv1, &value1) !=
1632 if (is_output_feature)
1634 if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1636 vnet_buffer (b1)->ip.reass.l4_src_port,
1637 vnet_buffer (b1)->ip.reass.l4_dst_port, thread_index,
1642 * Send DHCP packets to the ipv4 stack, or we won't
1643 * be able to use dhcp client on the outside interface
1646 (proto1 == NAT_PROTOCOL_UDP
1647 && (vnet_buffer (b1)->ip.reass.l4_dst_port ==
1648 clib_host_to_net_u16
1649 (UDP_DST_PORT_dhcp_to_server))
1650 && ip1->dst_address.as_u32 == 0xffffffff))
1655 if (PREDICT_FALSE (nat44_ei_not_translate (
1656 nm, node, rx_sw_if_index1, ip1, proto1, rx_fib_index1,
1661 next1 = slow_path (nm, b1, ip1, ip1->src_address,
1662 vnet_buffer (b1)->ip.reass.l4_src_port,
1663 rx_fib_index1, proto1, &s1, node, next1,
1665 if (PREDICT_FALSE (next1 == NAT44_EI_IN2OUT_NEXT_DROP))
1668 if (PREDICT_FALSE (!s1))
1673 next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1678 s1 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1679 nat_value_get_session_index (&value1));
1681 b1->flags |= VNET_BUFFER_F_IS_NATED;
1683 old_addr1 = ip1->src_address.as_u32;
1684 ip1->src_address = s1->out2in.addr;
1685 new_addr1 = ip1->src_address.as_u32;
1686 if (!is_output_feature)
1687 vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1689 sum1 = ip1->checksum;
1690 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1691 ip4_header_t, src_address /* changed member */ );
1692 ip1->checksum = ip_csum_fold (sum1);
1694 if (PREDICT_TRUE (proto1 == NAT_PROTOCOL_TCP))
1696 if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1698 old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1699 new_port1 = udp1->src_port = s1->out2in.port;
1700 sum1 = tcp1->checksum;
1701 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1703 dst_address /* changed member */ );
1704 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1705 ip4_header_t /* cheat */ ,
1706 length /* changed member */ );
1707 mss_clamping (nm->mss_clamping, tcp1, &sum1);
1708 tcp1->checksum = ip_csum_fold (sum1);
1710 vlib_increment_simple_counter (is_slow_path ?
1711 &nm->counters.slowpath.in2out.tcp :
1712 &nm->counters.fastpath.in2out.tcp,
1713 thread_index, cntr_sw_if_index1, 1);
1717 if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1719 udp1->src_port = s1->out2in.port;
1720 if (PREDICT_FALSE (udp1->checksum))
1722 old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1723 new_port1 = udp1->src_port;
1724 sum1 = udp1->checksum;
1725 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t, dst_address /* changed member */
1728 ip_csum_update (sum1, old_port1, new_port1,
1729 ip4_header_t /* cheat */ ,
1730 length /* changed member */ );
1731 udp1->checksum = ip_csum_fold (sum1);
1734 vlib_increment_simple_counter (is_slow_path ?
1735 &nm->counters.slowpath.in2out.udp :
1736 &nm->counters.fastpath.in2out.udp,
1737 thread_index, cntr_sw_if_index1, 1);
1741 nat44_ei_session_update_counters (
1742 s1, now, vlib_buffer_length_in_chain (vm, b1), thread_index);
1743 /* Per-user LRU list maintenance */
1744 nat44_ei_session_update_lru (nm, s1, thread_index);
1747 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1748 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1750 nat44_ei_in2out_trace_t *t =
1751 vlib_add_trace (vm, node, b1, sizeof (*t));
1752 t->sw_if_index = rx_sw_if_index1;
1753 t->next_index = next1;
1754 t->session_index = ~0;
1756 t->session_index = s1 - nm->per_thread_data[thread_index].sessions;
1759 if (next1 == NAT44_EI_IN2OUT_NEXT_DROP)
1761 vlib_increment_simple_counter (
1762 is_slow_path ? &nm->counters.slowpath.in2out.drops :
1763 &nm->counters.fastpath.in2out.drops,
1764 thread_index, cntr_sw_if_index1, 1);
1773 while (n_left_from > 0)
1777 u32 rx_sw_if_index0;
1778 u32 tx_sw_if_index0;
1779 u32 cntr_sw_if_index0;
1782 u32 new_addr0, old_addr0;
1783 u16 old_port0, new_port0;
1786 icmp46_header_t *icmp0;
1789 nat44_ei_session_t *s0 = 0;
1790 clib_bihash_kv_8_8_t kv0, value0;
1791 u32 iph_offset0 = 0;
1795 next0 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
1797 if (is_output_feature)
1798 iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1800 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1803 udp0 = ip4_next_header (ip0);
1804 tcp0 = (tcp_header_t *) udp0;
1805 icmp0 = (icmp46_header_t *) udp0;
1807 rx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1808 tx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1810 is_output_feature ? tx_sw_if_index0 : rx_sw_if_index0;
1812 vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index0);
1814 if (PREDICT_FALSE (ip0->ttl == 1))
1816 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1817 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1818 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1820 next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1824 proto0 = ip_proto_to_nat_proto (ip0->protocol);
1826 /* Next configured feature, probably ip4-lookup */
1829 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1831 if (nat_in2out_sm_unknown_proto (nm, b0, ip0, rx_fib_index0))
1833 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1835 node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1837 vlib_increment_simple_counter (
1838 is_slow_path ? &nm->counters.slowpath.in2out.other :
1839 &nm->counters.fastpath.in2out.other,
1840 thread_index, cntr_sw_if_index0, 1);
1844 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1846 next0 = nat44_ei_icmp_in2out_slow_path (
1847 nm, b0, ip0, icmp0, rx_sw_if_index0, rx_fib_index0, node,
1848 next0, now, thread_index, &s0);
1849 vlib_increment_simple_counter (
1850 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1851 &nm->counters.fastpath.in2out.icmp,
1852 thread_index, cntr_sw_if_index0, 1);
1858 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1860 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1864 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1866 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1871 init_nat_k (&kv0, ip0->src_address,
1872 vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1875 if (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
1879 if (is_output_feature)
1881 if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1883 vnet_buffer (b0)->ip.reass.l4_src_port,
1884 vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1889 * Send DHCP packets to the ipv4 stack, or we won't
1890 * be able to use dhcp client on the outside interface
1893 (proto0 == NAT_PROTOCOL_UDP
1894 && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1895 clib_host_to_net_u16
1896 (UDP_DST_PORT_dhcp_to_server))
1897 && ip0->dst_address.as_u32 == 0xffffffff))
1902 if (PREDICT_FALSE (nat44_ei_not_translate (
1903 nm, node, rx_sw_if_index0, ip0, proto0, rx_fib_index0,
1908 next0 = slow_path (nm, b0, ip0, ip0->src_address,
1909 vnet_buffer (b0)->ip.reass.l4_src_port,
1910 rx_fib_index0, proto0, &s0, node, next0,
1913 if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
1916 if (PREDICT_FALSE (!s0))
1921 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1926 s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1927 nat_value_get_session_index (&value0));
1929 b0->flags |= VNET_BUFFER_F_IS_NATED;
1931 old_addr0 = ip0->src_address.as_u32;
1932 ip0->src_address = s0->out2in.addr;
1933 new_addr0 = ip0->src_address.as_u32;
1934 if (!is_output_feature)
1935 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1937 sum0 = ip0->checksum;
1938 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1939 ip4_header_t, src_address /* changed member */ );
1940 ip0->checksum = ip_csum_fold (sum0);
1942 if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1944 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1946 old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1947 new_port0 = udp0->src_port = s0->out2in.port;
1948 sum0 = tcp0->checksum;
1950 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1951 dst_address /* changed member */ );
1953 ip_csum_update (sum0, old_port0, new_port0,
1954 ip4_header_t /* cheat */ ,
1955 length /* changed member */ );
1956 mss_clamping (nm->mss_clamping, tcp0, &sum0);
1957 tcp0->checksum = ip_csum_fold (sum0);
1959 vlib_increment_simple_counter (is_slow_path ?
1960 &nm->counters.slowpath.in2out.tcp :
1961 &nm->counters.fastpath.in2out.tcp,
1962 thread_index, cntr_sw_if_index0, 1);
1966 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1968 udp0->src_port = s0->out2in.port;
1969 if (PREDICT_FALSE (udp0->checksum))
1971 old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1972 new_port0 = udp0->src_port;
1973 sum0 = udp0->checksum;
1975 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1976 dst_address /* changed member */ );
1978 ip_csum_update (sum0, old_port0, new_port0,
1979 ip4_header_t /* cheat */ ,
1980 length /* changed member */ );
1981 udp0->checksum = ip_csum_fold (sum0);
1984 vlib_increment_simple_counter (is_slow_path ?
1985 &nm->counters.slowpath.in2out.udp :
1986 &nm->counters.fastpath.in2out.udp,
1987 thread_index, cntr_sw_if_index0, 1);
1991 nat44_ei_session_update_counters (
1992 s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1993 /* Per-user LRU list maintenance */
1994 nat44_ei_session_update_lru (nm, s0, thread_index);
1997 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1998 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2000 nat44_ei_in2out_trace_t *t =
2001 vlib_add_trace (vm, node, b0, sizeof (*t));
2002 t->is_slow_path = is_slow_path;
2003 t->sw_if_index = rx_sw_if_index0;
2004 t->next_index = next0;
2005 t->session_index = ~0;
2007 t->session_index = s0 - nm->per_thread_data[thread_index].sessions;
2010 if (next0 == NAT44_EI_IN2OUT_NEXT_DROP)
2012 vlib_increment_simple_counter (
2013 is_slow_path ? &nm->counters.slowpath.in2out.drops :
2014 &nm->counters.fastpath.in2out.drops,
2015 thread_index, cntr_sw_if_index0, 1);
2023 vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
2025 return frame->n_vectors;
2028 static_always_inline uword
2029 nat44_ei_in2out_hairpinning_finish_inline (vlib_main_t *vm,
2030 vlib_node_runtime_t *node,
2031 vlib_frame_t *frame)
2033 u32 n_left_from, *from, *to_next;
2034 u32 thread_index = vm->thread_index;
2035 nat44_ei_in2out_next_t next_index;
2036 nat44_ei_main_t *nm = &nat44_ei_main;
2037 int is_hairpinning = 0;
2039 from = vlib_frame_vector_args (frame);
2040 n_left_from = frame->n_vectors;
2041 next_index = node->cached_next_index;
2043 while (n_left_from > 0)
2047 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2049 while (n_left_from > 0 && n_left_to_next > 0)
2058 icmp46_header_t *icmp0;
2060 u32 required_thread_index = thread_index;
2067 n_left_to_next -= 1;
2069 b0 = vlib_get_buffer (vm, bi0);
2070 next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP;
2072 ip0 = vlib_buffer_get_current (b0);
2073 udp0 = ip4_next_header (ip0);
2074 tcp0 = (tcp_header_t *) udp0;
2075 icmp0 = (icmp46_header_t *) udp0;
2077 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2078 proto0 = ip_proto_to_nat_proto (ip0->protocol);
2082 case NAT_PROTOCOL_TCP:
2084 case NAT_PROTOCOL_UDP:
2085 is_hairpinning = nat44_ei_hairpinning (
2086 vm, node, nm, thread_index, b0, ip0, udp0, tcp0, proto0,
2087 0 /* do_trace */, &required_thread_index);
2089 case NAT_PROTOCOL_ICMP:
2090 is_hairpinning = (0 == nat44_ei_icmp_hairpinning (
2091 nm, b0, thread_index, ip0, icmp0,
2092 &required_thread_index));
2094 case NAT_PROTOCOL_OTHER:
2095 // this should never happen
2096 next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2100 if (thread_index != required_thread_index)
2102 // but we already did a handoff ...
2103 next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2106 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
2107 (b0->flags & VLIB_BUFFER_IS_TRACED)))
2109 nat44_ei_in2out_trace_t *t =
2110 vlib_add_trace (vm, node, b0, sizeof (*t));
2111 t->sw_if_index = sw_if_index0;
2112 t->next_index = next0;
2113 t->is_hairpinning = is_hairpinning;
2116 if (next0 != NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP)
2118 vlib_increment_simple_counter (
2119 &nm->counters.fastpath.in2out.other, sw_if_index0,
2120 vm->thread_index, 1);
2123 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2124 n_left_to_next, bi0, next0);
2127 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2130 return frame->n_vectors;
2133 VLIB_NODE_FN (nat44_ei_hairpinning_node)
2134 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2136 u32 n_left_from, *from, *to_next;
2137 u32 thread_index = vm->thread_index;
2138 nat44_ei_hairpin_next_t next_index;
2139 nat44_ei_main_t *nm = &nat44_ei_main;
2140 vnet_feature_main_t *fm = &feature_main;
2141 u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2142 vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2144 from = vlib_frame_vector_args (frame);
2145 n_left_from = frame->n_vectors;
2146 next_index = node->cached_next_index;
2148 while (n_left_from > 0)
2152 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2154 while (n_left_from > 0 && n_left_to_next > 0)
2164 u32 required_thread_index = thread_index;
2171 n_left_to_next -= 1;
2173 b0 = vlib_get_buffer (vm, bi0);
2174 ip0 = vlib_buffer_get_current (b0);
2175 udp0 = ip4_next_header (ip0);
2176 tcp0 = (tcp_header_t *) udp0;
2177 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2179 proto0 = ip_proto_to_nat_proto (ip0->protocol);
2180 int next0_resolved = 0;
2182 if (nat44_ei_hairpinning (vm, node, nm, thread_index, b0, ip0, udp0,
2183 tcp0, proto0, 1, &required_thread_index))
2185 next0 = NAT44_EI_HAIRPIN_NEXT_LOOKUP;
2189 if (thread_index != required_thread_index)
2191 vnet_buffer (b0)->snat.required_thread_index =
2192 required_thread_index;
2193 next0 = NAT44_EI_HAIRPIN_NEXT_HANDOFF;
2197 if (!next0_resolved)
2198 vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2201 if (next0 != NAT44_EI_HAIRPIN_NEXT_DROP)
2203 vlib_increment_simple_counter (
2204 &nm->counters.hairpinning, vm->thread_index, sw_if_index0, 1);
2207 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2208 n_left_to_next, bi0, next0);
2211 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2214 return frame->n_vectors;
2217 VLIB_NODE_FN (nat44_ei_in2out_node)
2218 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2220 return nat44_ei_in2out_node_fn_inline (vm, node, frame, 0, 0);
2223 VLIB_NODE_FN (nat44_ei_in2out_output_node)
2224 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2226 return nat44_ei_in2out_node_fn_inline (vm, node, frame, 0, 1);
2229 VLIB_NODE_FN (nat44_ei_in2out_slowpath_node)
2230 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2232 return nat44_ei_in2out_node_fn_inline (vm, node, frame, 1, 0);
2235 VLIB_NODE_FN (nat44_ei_in2out_output_slowpath_node)
2236 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2238 return nat44_ei_in2out_node_fn_inline (vm, node, frame, 1, 1);
2241 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_handoff_ip4_lookup_node)
2242 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2244 return nat44_ei_hairpinning_handoff_fn_inline (
2246 nat44_ei_main.in2out_hairpinning_finish_ip4_lookup_node_fq_index);
2249 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_handoff_interface_output_node)
2250 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2252 return nat44_ei_hairpinning_handoff_fn_inline (
2254 nat44_ei_main.in2out_hairpinning_finish_interface_output_node_fq_index);
2257 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_finish_ip4_lookup_node)
2258 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2260 return nat44_ei_in2out_hairpinning_finish_inline (vm, node, frame);
2263 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_finish_interface_output_node)
2264 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2266 return nat44_ei_in2out_hairpinning_finish_inline (vm, node, frame);
2269 VLIB_NODE_FN (nat44_ei_hairpinning_handoff_node)
2270 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2272 return nat44_ei_hairpinning_handoff_fn_inline (
2273 vm, node, frame, nat44_ei_main.hairpinning_fq_index);
2276 VLIB_REGISTER_NODE (nat44_ei_in2out_node) = {
2277 .name = "nat44-ei-in2out",
2278 .vector_size = sizeof (u32),
2279 .format_trace = format_nat44_ei_in2out_trace,
2280 .type = VLIB_NODE_TYPE_INTERNAL,
2281 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2282 .error_strings = nat44_ei_in2out_error_strings,
2283 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2284 .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
2286 [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
2287 [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2288 [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
2289 [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2290 [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2294 VLIB_REGISTER_NODE (nat44_ei_in2out_output_node) = {
2295 .name = "nat44-ei-in2out-output",
2296 .vector_size = sizeof (u32),
2297 .format_trace = format_nat44_ei_in2out_trace,
2298 .type = VLIB_NODE_TYPE_INTERNAL,
2299 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2300 .error_strings = nat44_ei_in2out_error_strings,
2301 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2302 .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
2304 [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
2305 [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "interface-output",
2306 [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-output-slowpath",
2307 [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2308 [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-interface-output",
2312 VLIB_REGISTER_NODE (nat44_ei_in2out_slowpath_node) = {
2313 .name = "nat44-ei-in2out-slowpath",
2314 .vector_size = sizeof (u32),
2315 .format_trace = format_nat44_ei_in2out_trace,
2316 .type = VLIB_NODE_TYPE_INTERNAL,
2317 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2318 .error_strings = nat44_ei_in2out_error_strings,
2319 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2320 .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
2322 [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
2323 [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2324 [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
2325 [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2326 [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2330 VLIB_REGISTER_NODE (nat44_ei_in2out_output_slowpath_node) = {
2331 .name = "nat44-ei-in2out-output-slowpath",
2332 .vector_size = sizeof (u32),
2333 .format_trace = format_nat44_ei_in2out_trace,
2334 .type = VLIB_NODE_TYPE_INTERNAL,
2335 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2336 .error_strings = nat44_ei_in2out_error_strings,
2337 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2338 .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
2340 [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
2341 [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "interface-output",
2342 [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-output-slowpath",
2343 [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2344 [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-interface-output",
2348 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_handoff_ip4_lookup_node) = {
2349 .name = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2350 .vector_size = sizeof (u32),
2351 .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2352 .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2353 .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2360 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_handoff_interface_output_node) = {
2361 .name = "nat44-ei-in2out-hairpinning-handoff-interface-output",
2362 .vector_size = sizeof (u32),
2363 .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2364 .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2365 .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2372 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_finish_ip4_lookup_node) = {
2373 .name = "nat44-ei-in2out-hairpinning-finish-ip4-lookup",
2374 .vector_size = sizeof (u32),
2375 .format_trace = format_nat44_ei_in2out_fast_trace,
2376 .type = VLIB_NODE_TYPE_INTERNAL,
2377 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2378 .error_strings = nat44_ei_in2out_error_strings,
2379 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2380 .n_next_nodes = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2382 [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2383 [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "ip4-lookup",
2387 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_finish_interface_output_node) = {
2388 .name = "nat44-ei-in2out-hairpinning-finish-interface-output",
2389 .vector_size = sizeof (u32),
2390 .format_trace = format_nat44_ei_in2out_fast_trace,
2391 .type = VLIB_NODE_TYPE_INTERNAL,
2392 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2393 .error_strings = nat44_ei_in2out_error_strings,
2394 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2395 .n_next_nodes = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2397 [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2398 [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "interface-output",
2402 VLIB_REGISTER_NODE (nat44_ei_hairpinning_handoff_node) = {
2403 .name = "nat44-ei-hairpinning-handoff",
2404 .vector_size = sizeof (u32),
2405 .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2406 .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2407 .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2414 VLIB_REGISTER_NODE (nat44_ei_hairpinning_node) = {
2415 .name = "nat44-ei-hairpinning",
2416 .vector_size = sizeof (u32),
2417 .type = VLIB_NODE_TYPE_INTERNAL,
2418 .format_trace = format_nat44_ei_hairpin_trace,
2419 .n_next_nodes = NAT44_EI_HAIRPIN_N_NEXT,
2421 [NAT44_EI_HAIRPIN_NEXT_DROP] = "error-drop",
2422 [NAT44_EI_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
2423 [NAT44_EI_HAIRPIN_NEXT_HANDOFF] = "nat44-ei-hairpinning-handoff",
2428 * fd.io coding-style-patch-verification: ON
2431 * eval: (c-set-style "gnu")