2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
17 * @brief NAT44 EI inside to outside network translation
20 #include <vlib/vlib.h>
22 #include <vnet/vnet.h>
23 #include <vnet/ip/ip.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/udp/udp_local.h>
26 #include <vnet/fib/ip4_fib.h>
28 #include <vppinfra/hash.h>
29 #include <vppinfra/error.h>
31 #include <nat/lib/log.h>
32 #include <nat/lib/nat_syslog.h>
33 #include <nat/lib/ipfix_logging.h>
34 #include <nat/lib/nat_inlines.h>
35 #include <nat/nat44-ei/nat44_ei_inlines.h>
36 #include <nat/nat44-ei/nat44_ei.h>
37 #include <nat/nat44-ei/nat44_ei_hairpinning.h>
46 } nat44_ei_in2out_trace_t;
48 /* packet trace format function */
50 format_nat44_ei_in2out_trace (u8 *s, va_list *args)
52 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
53 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
54 nat44_ei_in2out_trace_t *t = va_arg (*args, nat44_ei_in2out_trace_t *);
57 tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
59 s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
60 t->sw_if_index, t->next_index, t->session_index);
61 if (t->is_hairpinning)
63 s = format (s, ", with-hairpinning");
70 format_nat44_ei_in2out_fast_trace (u8 *s, va_list *args)
72 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
73 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
74 nat44_ei_in2out_trace_t *t = va_arg (*args, nat44_ei_in2out_trace_t *);
76 s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
77 t->sw_if_index, t->next_index);
82 #define foreach_nat44_ei_in2out_error \
83 _ (UNSUPPORTED_PROTOCOL, "unsupported protocol") \
84 _ (OUT_OF_PORTS, "out of ports") \
85 _ (BAD_OUTSIDE_FIB, "outside VRF ID not found") \
86 _ (BAD_ICMP_TYPE, "unsupported ICMP type") \
87 _ (NO_TRANSLATION, "no translation") \
88 _ (MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded") \
89 _ (CANNOT_CREATE_USER, "cannot create NAT user")
93 #define _(sym, str) NAT44_EI_IN2OUT_ERROR_##sym,
94 foreach_nat44_ei_in2out_error
96 NAT44_EI_IN2OUT_N_ERROR,
97 } nat44_ei_in2out_error_t;
99 static char *nat44_ei_in2out_error_strings[] = {
100 #define _(sym,string) string,
101 foreach_nat44_ei_in2out_error
107 NAT44_EI_IN2OUT_NEXT_LOOKUP,
108 NAT44_EI_IN2OUT_NEXT_DROP,
109 NAT44_EI_IN2OUT_NEXT_ICMP_ERROR,
110 NAT44_EI_IN2OUT_NEXT_SLOW_PATH,
111 NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF,
112 NAT44_EI_IN2OUT_N_NEXT,
113 } nat44_ei_in2out_next_t;
117 NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP,
118 NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP,
119 NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
120 } nat44_ei_in2out_hairpinnig_finish_next_t;
123 nat44_ei_not_translate_fast (vlib_node_runtime_t *node, u32 sw_if_index0,
124 ip4_header_t *ip0, u32 proto0, u32 rx_fib_index0)
126 nat44_ei_main_t *nm = &nat44_ei_main;
131 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
132 nat44_ei_outside_fib_t *outside_fib;
134 .fp_proto = FIB_PROTOCOL_IP4,
137 .ip4.as_u32 = ip0->dst_address.as_u32,
142 /* Don't NAT packet aimed at the intfc address */
143 if (PREDICT_FALSE (nat44_ei_is_interface_addr (
144 nm->ip4_main, node, sw_if_index0, ip0->dst_address.as_u32)))
147 fei = fib_table_lookup (rx_fib_index0, &pfx);
148 if (FIB_NODE_INDEX_INVALID != fei)
150 u32 sw_if_index = fib_entry_get_resolving_interface (fei);
151 if (sw_if_index == ~0)
153 vec_foreach (outside_fib, nm->outside_fibs)
155 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
156 if (FIB_NODE_INDEX_INVALID != fei)
158 sw_if_index = fib_entry_get_resolving_interface (fei);
159 if (sw_if_index != ~0)
164 if (sw_if_index == ~0)
167 nat44_ei_interface_t *i;
168 pool_foreach (i, nm->interfaces)
170 /* NAT packet aimed at outside interface */
171 if ((nat44_ei_interface_is_outside (i)) &&
172 (sw_if_index == i->sw_if_index))
181 nat44_ei_not_translate (nat44_ei_main_t *nm, vlib_node_runtime_t *node,
182 u32 sw_if_index0, ip4_header_t *ip0, u32 proto0,
183 u32 rx_fib_index0, u32 thread_index)
185 udp_header_t *udp0 = ip4_next_header (ip0);
186 clib_bihash_kv_8_8_t kv0, value0;
188 init_nat_k (&kv0, ip0->dst_address, udp0->dst_port, nm->outside_fib_index,
191 /* NAT packet aimed at external address if */
192 /* has active sessions */
193 if (clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
195 /* or is static mappings */
196 ip4_address_t placeholder_addr;
197 u16 placeholder_port;
198 u32 placeholder_fib_index;
199 if (!nat44_ei_static_mapping_match (ip0->dst_address, udp0->dst_port,
200 nm->outside_fib_index, proto0,
201 &placeholder_addr, &placeholder_port,
202 &placeholder_fib_index, 1, 0, 0))
208 if (nm->forwarding_enabled)
211 return nat44_ei_not_translate_fast (node, sw_if_index0, ip0, proto0,
216 nat44_ei_not_translate_output_feature (nat44_ei_main_t *nm, ip4_header_t *ip0,
217 u32 proto0, u16 src_port, u16 dst_port,
218 u32 thread_index, u32 sw_if_index)
220 clib_bihash_kv_8_8_t kv0, value0;
221 nat44_ei_interface_t *i;
224 init_nat_k (&kv0, ip0->src_address, src_port,
225 ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
227 if (!clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
231 init_nat_k (&kv0, ip0->dst_address, dst_port,
232 ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
233 if (!clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
236 pool_foreach (i, nm->output_feature_interfaces)
238 if ((nat44_ei_interface_is_inside (i)) &&
239 (sw_if_index == i->sw_if_index))
248 #ifndef CLIB_MARCH_VARIANT
250 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
252 nat44_ei_main_t *nm = &nat44_ei_main;
253 nat44_ei_is_idle_session_ctx_t *ctx = arg;
254 nat44_ei_session_t *s;
255 u64 sess_timeout_time;
256 nat44_ei_main_per_thread_data_t *tnm =
257 vec_elt_at_index (nm->per_thread_data, ctx->thread_index);
258 clib_bihash_kv_8_8_t s_kv;
260 if (ctx->thread_index != nat_value_get_thread_index (kv))
265 s = pool_elt_at_index (tnm->sessions, nat_value_get_session_index (kv));
266 sess_timeout_time = s->last_heard + (f64) nat_session_get_timeout (
267 &nm->timeouts, s->nat_proto, s->state);
268 if (ctx->now >= sess_timeout_time)
270 init_nat_o2i_k (&s_kv, s);
271 if (clib_bihash_add_del_8_8 (&nm->out2in, &s_kv, 0))
272 nat_elog_warn (nm, "out2in key del failed");
274 nat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
275 s->in2out.addr.as_u32,
276 s->out2in.addr.as_u32,
280 s->in2out.fib_index);
282 nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
283 &s->in2out.addr, s->in2out.port,
284 &s->out2in.addr, s->out2in.port, s->nat_proto);
286 nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
287 s->ext_host_port, s->nat_proto, s->out2in.fib_index,
290 if (!nat44_ei_is_session_static (s))
291 nat44_ei_free_outside_address_and_port (
292 nm->addresses, ctx->thread_index, &s->out2in.addr, s->out2in.port,
295 nat44_ei_delete_session (nm, s, ctx->thread_index);
304 slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0, ip4_header_t *ip0,
305 ip4_address_t i2o_addr, u16 i2o_port, u32 rx_fib_index0,
306 nat_protocol_t nat_proto, nat44_ei_session_t **sessionp,
307 vlib_node_runtime_t *node, u32 next0, u32 thread_index, f64 now)
310 nat44_ei_session_t *s = 0;
311 clib_bihash_kv_8_8_t kv0;
313 nat44_ei_outside_fib_t *outside_fib;
314 fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
317 .fp_proto = FIB_PROTOCOL_IP4,
320 .ip4.as_u32 = ip0->dst_address.as_u32,
323 nat44_ei_is_idle_session_ctx_t ctx0;
324 ip4_address_t sm_addr;
328 if (PREDICT_FALSE (nat44_ei_maximum_sessions_exceeded (nm, thread_index)))
330 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
331 nat_ipfix_logging_max_sessions (thread_index,
332 nm->max_translations_per_thread);
333 nat_elog_notice (nm, "maximum sessions exceeded");
334 return NAT44_EI_IN2OUT_NEXT_DROP;
337 /* First try to match static mapping by local address and port */
338 if (nat44_ei_static_mapping_match (i2o_addr, i2o_port, rx_fib_index0,
339 nat_proto, &sm_addr, &sm_port,
340 &sm_fib_index, 0, 0, &identity_nat))
342 /* Try to create dynamic translation */
343 if (nm->alloc_addr_and_port (
344 nm->addresses, rx_fib_index0, thread_index, nat_proto,
345 ip0->src_address, &sm_addr, &sm_port, nm->port_per_thread,
346 nm->per_thread_data[thread_index].snat_thread_index))
348 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_OUT_OF_PORTS];
349 return NAT44_EI_IN2OUT_NEXT_DROP;
354 if (PREDICT_FALSE (identity_nat))
363 u = nat44_ei_user_get_or_create (nm, &ip0->src_address, rx_fib_index0,
367 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_CANNOT_CREATE_USER];
368 return NAT44_EI_IN2OUT_NEXT_DROP;
371 s = nat44_ei_session_alloc_or_recycle (nm, u, thread_index, now);
374 nat44_ei_delete_user_with_no_session (nm, u, thread_index);
375 nat_elog_warn (nm, "create NAT session failed");
376 return NAT44_EI_IN2OUT_NEXT_DROP;
380 s->flags |= NAT44_EI_SESSION_FLAG_STATIC_MAPPING;
381 nat44_ei_user_session_increment (nm, u, is_sm);
382 s->in2out.addr = i2o_addr;
383 s->in2out.port = i2o_port;
384 s->in2out.fib_index = rx_fib_index0;
385 s->nat_proto = nat_proto;
386 s->out2in.addr = sm_addr;
387 s->out2in.port = sm_port;
388 s->out2in.fib_index = nm->outside_fib_index;
389 switch (vec_len (nm->outside_fibs))
392 s->out2in.fib_index = nm->outside_fib_index;
395 s->out2in.fib_index = nm->outside_fibs[0].fib_index;
398 vec_foreach (outside_fib, nm->outside_fibs)
400 fei = fib_table_lookup (outside_fib->fib_index, &pfx);
401 if (FIB_NODE_INDEX_INVALID != fei)
403 if (fib_entry_get_resolving_interface (fei) != ~0)
405 s->out2in.fib_index = outside_fib->fib_index;
412 s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
413 s->ext_host_port = vnet_buffer (b0)->ip.reass.l4_dst_port;
416 /* Add to translation hashes */
418 ctx0.thread_index = thread_index;
419 init_nat_i2o_kv (&kv0, s, thread_index,
420 s - nm->per_thread_data[thread_index].sessions);
421 if (clib_bihash_add_or_overwrite_stale_8_8 (
422 &nm->in2out, &kv0, nat44_i2o_is_idle_session_cb, &ctx0))
423 nat_elog_notice (nm, "in2out key add failed");
425 init_nat_o2i_kv (&kv0, s, thread_index,
426 s - nm->per_thread_data[thread_index].sessions);
427 if (clib_bihash_add_or_overwrite_stale_8_8 (
428 &nm->out2in, &kv0, nat44_o2i_is_idle_session_cb, &ctx0))
429 nat_elog_notice (nm, "out2in key add failed");
432 nat_ipfix_logging_nat44_ses_create (
433 thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->nat_proto,
434 s->in2out.port, s->out2in.port, s->in2out.fib_index);
436 nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index, &s->in2out.addr,
437 s->in2out.port, &s->out2in.addr, s->out2in.port,
440 nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
441 s->out2in.port, &s->ext_host_addr, s->ext_host_port,
442 &s->ext_host_nat_addr, s->ext_host_nat_port, s->nat_proto,
443 s->in2out.fib_index, s->flags, thread_index, 0);
448 #ifndef CLIB_MARCH_VARIANT
449 static_always_inline nat44_ei_in2out_error_t
450 icmp_get_key (vlib_buffer_t *b, ip4_header_t *ip0, ip4_address_t *addr,
451 u16 *port, nat_protocol_t *nat_proto)
453 icmp46_header_t *icmp0;
454 icmp_echo_header_t *echo0, *inner_echo0 = 0;
455 ip4_header_t *inner_ip0 = 0;
457 icmp46_header_t *inner_icmp0;
459 icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
460 echo0 = (icmp_echo_header_t *) (icmp0 + 1);
462 if (!icmp_type_is_error_message
463 (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
465 *nat_proto = NAT_PROTOCOL_ICMP;
466 *addr = ip0->src_address;
467 *port = vnet_buffer (b)->ip.reass.l4_src_port;
471 inner_ip0 = (ip4_header_t *) (echo0 + 1);
472 l4_header = ip4_next_header (inner_ip0);
473 *nat_proto = ip_proto_to_nat_proto (inner_ip0->protocol);
474 *addr = inner_ip0->dst_address;
477 case NAT_PROTOCOL_ICMP:
478 inner_icmp0 = (icmp46_header_t *) l4_header;
479 inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
480 *port = inner_echo0->identifier;
482 case NAT_PROTOCOL_UDP:
483 case NAT_PROTOCOL_TCP:
484 *port = ((tcp_udp_header_t *) l4_header)->dst_port;
487 return NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
490 return -1; /* success */
494 * Get address and port values to be used for ICMP packet translation
495 * and create session if needed
497 * @param[in,out] nm NAT main
498 * @param[in,out] node NAT node runtime
499 * @param[in] thread_index thread index
500 * @param[in,out] b0 buffer containing packet to be translated
501 * @param[in,out] ip0 ip header
502 * @param[out] p_proto protocol used for matching
503 * @param[out] p_value address and port after NAT translation
504 * @param[out] p_dont_translate if packet should not be translated
505 * @param d optional parameter
506 * @param e optional parameter
509 nat44_ei_icmp_match_in2out_slow (vlib_node_runtime_t *node, u32 thread_index,
510 vlib_buffer_t *b0, ip4_header_t *ip0,
511 ip4_address_t *addr, u16 *port,
512 u32 *fib_index, nat_protocol_t *proto,
513 nat44_ei_session_t **p_s0, u8 *dont_translate)
515 nat44_ei_main_t *nm = &nat44_ei_main;
516 nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
518 nat44_ei_session_t *s0 = 0;
519 clib_bihash_kv_8_8_t kv0, value0;
522 vlib_main_t *vm = vlib_get_main ();
525 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
526 *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
528 err = icmp_get_key (b0, ip0, addr, port, proto);
531 b0->error = node->errors[err];
532 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
536 init_nat_k (&kv0, *addr, *port, *fib_index, *proto);
537 if (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
539 if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
541 if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
542 nm, ip0, *proto, *port, *port, thread_index, sw_if_index0)))
550 if (PREDICT_FALSE (nat44_ei_not_translate (
551 nm, node, sw_if_index0, ip0, NAT_PROTOCOL_ICMP, *fib_index,
560 (icmp_type_is_error_message
561 (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags)))
563 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
564 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
568 next0 = slow_path (nm, b0, ip0, *addr, *port, *fib_index, *proto, &s0,
569 node, next0, thread_index, vlib_time_now (vm));
571 if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
583 (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
585 && vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
587 && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
588 reass.icmp_type_or_tcp_flags)))
590 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
591 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
595 s0 = pool_elt_at_index (tnm->sessions,
596 nat_value_get_session_index (&value0));
602 *addr = s0->out2in.addr;
603 *port = s0->out2in.port;
604 *fib_index = s0->out2in.fib_index;
612 #ifndef CLIB_MARCH_VARIANT
614 nat44_ei_icmp_match_in2out_fast (vlib_node_runtime_t *node, u32 thread_index,
615 vlib_buffer_t *b0, ip4_header_t *ip0,
616 ip4_address_t *addr, u16 *port,
617 u32 *fib_index, nat_protocol_t *proto,
618 nat44_ei_session_t **s0, u8 *dont_translate)
626 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
627 *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
629 err = icmp_get_key (b0, ip0, addr, port, proto);
632 b0->error = node->errors[err];
633 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
637 ip4_address_t sm_addr;
641 if (nat44_ei_static_mapping_match (*addr, *port, *fib_index, *proto,
642 &sm_addr, &sm_port, &sm_fib_index, 0,
645 if (PREDICT_FALSE (nat44_ei_not_translate_fast (
646 node, sw_if_index0, ip0, IP_PROTOCOL_ICMP, *fib_index)))
652 if (icmp_type_is_error_message
653 (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
655 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
659 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_NO_TRANSLATION];
660 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
665 (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags != ICMP4_echo_request
666 && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
667 ICMP4_echo_reply || !is_addr_only)
668 && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
669 reass.icmp_type_or_tcp_flags)))
671 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
672 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
681 u32 nat44_ei_icmp_in2out (vlib_buffer_t *b0, ip4_header_t *ip0,
682 icmp46_header_t *icmp0, u32 sw_if_index0,
683 u32 rx_fib_index0, vlib_node_runtime_t *node,
684 u32 next0, u32 thread_index,
685 nat44_ei_session_t **p_s0);
687 #ifndef CLIB_MARCH_VARIANT
689 nat44_ei_icmp_in2out (vlib_buffer_t *b0, ip4_header_t *ip0,
690 icmp46_header_t *icmp0, u32 sw_if_index0,
691 u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0,
692 u32 thread_index, nat44_ei_session_t **p_s0)
694 nat44_ei_main_t *nm = &nat44_ei_main;
695 vlib_main_t *vm = vlib_get_main ();
699 nat_protocol_t proto;
700 icmp_echo_header_t *echo0, *inner_echo0 = 0;
701 ip4_header_t *inner_ip0;
703 icmp46_header_t *inner_icmp0;
705 u32 new_addr0, old_addr0;
706 u16 old_id0, new_id0;
707 u16 old_checksum0, new_checksum0;
711 u32 required_thread_index = thread_index;
713 echo0 = (icmp_echo_header_t *) (icmp0 + 1);
715 if (PREDICT_TRUE (nm->pat))
717 next0_tmp = nat44_ei_icmp_match_in2out_slow (
718 node, thread_index, b0, ip0, &addr, &port, &fib_index, &proto, p_s0,
723 next0_tmp = nat44_ei_icmp_match_in2out_fast (
724 node, thread_index, b0, ip0, &addr, &port, &fib_index, &proto, p_s0,
730 if (next0 == NAT44_EI_IN2OUT_NEXT_DROP || dont_translate)
733 if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
736 ip_incremental_checksum_buffer (vm, b0,
738 (u8 *) vlib_buffer_get_current (b0),
739 ntohs (ip0->length) -
740 ip4_header_bytes (ip0), 0);
741 checksum0 = ~ip_csum_fold (sum0);
742 if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
744 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
749 old_addr0 = ip0->src_address.as_u32;
750 new_addr0 = ip0->src_address.as_u32 = addr.as_u32;
752 sum0 = ip0->checksum;
753 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
754 src_address /* changed member */ );
755 ip0->checksum = ip_csum_fold (sum0);
757 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
759 if (icmp0->checksum == 0)
760 icmp0->checksum = 0xffff;
762 if (!icmp_type_is_error_message (icmp0->type))
765 if (PREDICT_FALSE (new_id0 != echo0->identifier))
767 old_id0 = echo0->identifier;
769 echo0->identifier = new_id0;
771 sum0 = icmp0->checksum;
773 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
775 icmp0->checksum = ip_csum_fold (sum0);
780 inner_ip0 = (ip4_header_t *) (echo0 + 1);
781 l4_header = ip4_next_header (inner_ip0);
783 if (!ip4_header_checksum_is_valid (inner_ip0))
785 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
789 /* update inner destination IP address */
790 old_addr0 = inner_ip0->dst_address.as_u32;
791 inner_ip0->dst_address = addr;
792 new_addr0 = inner_ip0->dst_address.as_u32;
793 sum0 = icmp0->checksum;
794 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
795 dst_address /* changed member */ );
796 icmp0->checksum = ip_csum_fold (sum0);
798 /* update inner IP header checksum */
799 old_checksum0 = inner_ip0->checksum;
800 sum0 = inner_ip0->checksum;
801 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
802 dst_address /* changed member */ );
803 inner_ip0->checksum = ip_csum_fold (sum0);
804 new_checksum0 = inner_ip0->checksum;
805 sum0 = icmp0->checksum;
807 ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
809 icmp0->checksum = ip_csum_fold (sum0);
813 case NAT_PROTOCOL_ICMP:
814 inner_icmp0 = (icmp46_header_t *) l4_header;
815 inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
817 old_id0 = inner_echo0->identifier;
819 inner_echo0->identifier = new_id0;
821 sum0 = icmp0->checksum;
823 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
825 icmp0->checksum = ip_csum_fold (sum0);
827 case NAT_PROTOCOL_UDP:
828 case NAT_PROTOCOL_TCP:
829 old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
831 ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
833 sum0 = icmp0->checksum;
834 sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
836 icmp0->checksum = ip_csum_fold (sum0);
844 if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
846 if (0 != nat44_ei_icmp_hairpinning (nm, b0, thread_index, ip0, icmp0,
847 &required_thread_index))
848 vnet_buffer (b0)->sw_if_index[VLIB_TX] = fib_index;
849 if (thread_index != required_thread_index)
851 vnet_buffer (b0)->snat.required_thread_index = required_thread_index;
852 next0 = NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF;
861 static_always_inline u32
862 nat44_ei_icmp_in2out_slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0,
863 ip4_header_t *ip0, icmp46_header_t *icmp0,
864 u32 sw_if_index0, u32 rx_fib_index0,
865 vlib_node_runtime_t *node, u32 next0, f64 now,
866 u32 thread_index, nat44_ei_session_t **p_s0)
868 vlib_main_t *vm = vlib_get_main ();
870 next0 = nat44_ei_icmp_in2out (b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
871 node, next0, thread_index, p_s0);
872 nat44_ei_session_t *s0 = *p_s0;
873 if (PREDICT_TRUE (next0 != NAT44_EI_IN2OUT_NEXT_DROP && s0))
876 nat44_ei_session_update_counters (
877 s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
878 /* Per-user LRU list maintenance */
879 nat44_ei_session_update_lru (nm, s0, thread_index);
885 nat_in2out_sm_unknown_proto (nat44_ei_main_t *nm, vlib_buffer_t *b,
886 ip4_header_t *ip, u32 rx_fib_index)
888 clib_bihash_kv_8_8_t kv, value;
889 nat44_ei_static_mapping_t *m;
890 u32 old_addr, new_addr;
893 init_nat_k (&kv, ip->src_address, 0, rx_fib_index, 0);
894 if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
897 m = pool_elt_at_index (nm->static_mappings, value.value);
899 old_addr = ip->src_address.as_u32;
900 new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
902 sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
903 ip->checksum = ip_csum_fold (sum);
907 if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
909 vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
910 nat44_ei_hairpinning_sm_unknown_proto (nm, b, ip);
917 nat44_ei_in2out_node_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
918 vlib_frame_t *frame, int is_slow_path,
919 int is_output_feature)
921 u32 n_left_from, *from;
922 nat44_ei_main_t *nm = &nat44_ei_main;
923 f64 now = vlib_time_now (vm);
924 u32 thread_index = vm->thread_index;
926 from = vlib_frame_vector_args (frame);
927 n_left_from = frame->n_vectors;
929 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
930 u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
931 vlib_get_buffers (vm, from, b, n_left_from);
933 while (n_left_from >= 2)
935 vlib_buffer_t *b0, *b1;
937 u32 rx_sw_if_index0, rx_sw_if_index1;
938 u32 tx_sw_if_index0, tx_sw_if_index1;
939 u32 cntr_sw_if_index0, cntr_sw_if_index1;
940 ip4_header_t *ip0, *ip1;
941 ip_csum_t sum0, sum1;
942 u32 new_addr0, old_addr0, new_addr1, old_addr1;
943 u16 old_port0, new_port0, old_port1, new_port1;
944 udp_header_t *udp0, *udp1;
945 tcp_header_t *tcp0, *tcp1;
946 icmp46_header_t *icmp0, *icmp1;
947 u32 rx_fib_index0, rx_fib_index1;
949 nat44_ei_session_t *s0 = 0, *s1 = 0;
950 clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
951 u32 iph_offset0 = 0, iph_offset1 = 0;
958 /* Prefetch next iteration. */
959 if (PREDICT_TRUE (n_left_from >= 4))
961 vlib_buffer_t *p2, *p3;
966 vlib_prefetch_buffer_header (p2, LOAD);
967 vlib_prefetch_buffer_header (p3, LOAD);
969 clib_prefetch_load (p2->data);
970 clib_prefetch_load (p3->data);
973 if (is_output_feature)
974 iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
976 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
979 udp0 = ip4_next_header (ip0);
980 tcp0 = (tcp_header_t *) udp0;
981 icmp0 = (icmp46_header_t *) udp0;
983 rx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
984 tx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
986 is_output_feature ? tx_sw_if_index0 : rx_sw_if_index0;
988 vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index0);
990 next0 = next1 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
992 if (PREDICT_FALSE (ip0->ttl == 1))
994 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
995 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
996 ICMP4_time_exceeded_ttl_exceeded_in_transit,
998 next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1002 proto0 = ip_proto_to_nat_proto (ip0->protocol);
1004 /* Next configured feature, probably ip4-lookup */
1007 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1009 if (nat_in2out_sm_unknown_proto (nm, b0, ip0, rx_fib_index0))
1011 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1013 node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1015 vlib_increment_simple_counter (
1016 is_slow_path ? &nm->counters.slowpath.in2out.other :
1017 &nm->counters.fastpath.in2out.other,
1018 thread_index, cntr_sw_if_index0, 1);
1022 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1024 next0 = nat44_ei_icmp_in2out_slow_path (
1025 nm, b0, ip0, icmp0, rx_sw_if_index0, rx_fib_index0, node,
1026 next0, now, thread_index, &s0);
1027 vlib_increment_simple_counter (
1028 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1029 &nm->counters.fastpath.in2out.icmp,
1030 thread_index, cntr_sw_if_index0, 1);
1036 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1038 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1042 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1044 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1049 init_nat_k (&kv0, ip0->src_address,
1050 vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1052 if (PREDICT_FALSE (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0) !=
1057 if (is_output_feature)
1059 if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1061 vnet_buffer (b0)->ip.reass.l4_src_port,
1062 vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1067 * Send DHCP packets to the ipv4 stack, or we won't
1068 * be able to use dhcp client on the outside interface
1071 (proto0 == NAT_PROTOCOL_UDP
1072 && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1073 clib_host_to_net_u16
1074 (UDP_DST_PORT_dhcp_to_server))
1075 && ip0->dst_address.as_u32 == 0xffffffff))
1080 if (PREDICT_FALSE (nat44_ei_not_translate (
1081 nm, node, rx_sw_if_index0, ip0, proto0, rx_fib_index0,
1086 next0 = slow_path (nm, b0, ip0, ip0->src_address,
1087 vnet_buffer (b0)->ip.reass.l4_src_port,
1088 rx_fib_index0, proto0, &s0, node, next0,
1090 if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
1093 if (PREDICT_FALSE (!s0))
1098 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1103 s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1104 nat_value_get_session_index (&value0));
1106 b0->flags |= VNET_BUFFER_F_IS_NATED;
1108 old_addr0 = ip0->src_address.as_u32;
1109 ip0->src_address = s0->out2in.addr;
1110 new_addr0 = ip0->src_address.as_u32;
1111 if (!is_output_feature)
1112 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1114 sum0 = ip0->checksum;
1115 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1116 ip4_header_t, src_address /* changed member */ );
1117 ip0->checksum = ip_csum_fold (sum0);
1120 if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1122 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1124 old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1125 new_port0 = udp0->src_port = s0->out2in.port;
1126 sum0 = tcp0->checksum;
1127 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1129 dst_address /* changed member */ );
1130 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1131 ip4_header_t /* cheat */ ,
1132 length /* changed member */ );
1133 mss_clamping (nm->mss_clamping, tcp0, &sum0);
1134 tcp0->checksum = ip_csum_fold (sum0);
1136 vlib_increment_simple_counter (is_slow_path ?
1137 &nm->counters.slowpath.in2out.tcp :
1138 &nm->counters.fastpath.in2out.tcp,
1139 thread_index, cntr_sw_if_index0, 1);
1143 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1145 udp0->src_port = s0->out2in.port;
1146 if (PREDICT_FALSE (udp0->checksum))
1148 old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1149 new_port0 = udp0->src_port;
1150 sum0 = udp0->checksum;
1151 sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address /* changed member */
1154 ip_csum_update (sum0, old_port0, new_port0,
1155 ip4_header_t /* cheat */ ,
1156 length /* changed member */ );
1157 udp0->checksum = ip_csum_fold (sum0);
1160 vlib_increment_simple_counter (is_slow_path ?
1161 &nm->counters.slowpath.in2out.udp :
1162 &nm->counters.fastpath.in2out.udp,
1163 thread_index, cntr_sw_if_index0, 1);
1167 nat44_ei_session_update_counters (
1168 s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1169 /* Per-user LRU list maintenance */
1170 nat44_ei_session_update_lru (nm, s0, thread_index);
1173 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1174 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1176 nat44_ei_in2out_trace_t *t =
1177 vlib_add_trace (vm, node, b0, sizeof (*t));
1178 t->is_slow_path = is_slow_path;
1179 t->sw_if_index = rx_sw_if_index0;
1180 t->next_index = next0;
1181 t->session_index = ~0;
1183 t->session_index = s0 - nm->per_thread_data[thread_index].sessions;
1186 if (next0 == NAT44_EI_IN2OUT_NEXT_DROP)
1188 vlib_increment_simple_counter (
1189 is_slow_path ? &nm->counters.slowpath.in2out.drops :
1190 &nm->counters.fastpath.in2out.drops,
1191 thread_index, cntr_sw_if_index0, 1);
1194 if (is_output_feature)
1195 iph_offset1 = vnet_buffer (b1)->ip.reass.save_rewrite_length;
1197 ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1200 udp1 = ip4_next_header (ip1);
1201 tcp1 = (tcp_header_t *) udp1;
1202 icmp1 = (icmp46_header_t *) udp1;
1204 rx_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1205 tx_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_TX];
1207 is_output_feature ? tx_sw_if_index1 : rx_sw_if_index1;
1209 vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index1);
1211 if (PREDICT_FALSE (ip1->ttl == 1))
1213 vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1214 icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1215 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1217 next1 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1221 proto1 = ip_proto_to_nat_proto (ip1->protocol);
1223 /* Next configured feature, probably ip4-lookup */
1226 if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1228 if (nat_in2out_sm_unknown_proto (nm, b1, ip1, rx_fib_index1))
1230 next1 = NAT44_EI_IN2OUT_NEXT_DROP;
1232 node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1234 vlib_increment_simple_counter (
1235 is_slow_path ? &nm->counters.slowpath.in2out.other :
1236 &nm->counters.fastpath.in2out.other,
1237 thread_index, cntr_sw_if_index1, 1);
1241 if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1243 next1 = nat44_ei_icmp_in2out_slow_path (
1244 nm, b1, ip1, icmp1, rx_sw_if_index1, rx_fib_index1, node,
1245 next1, now, thread_index, &s1);
1246 vlib_increment_simple_counter (
1247 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1248 &nm->counters.fastpath.in2out.icmp,
1249 thread_index, cntr_sw_if_index1, 1);
1255 if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1257 next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1261 if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1263 next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1268 init_nat_k (&kv1, ip1->src_address,
1269 vnet_buffer (b1)->ip.reass.l4_src_port, rx_fib_index1,
1271 if (PREDICT_FALSE (clib_bihash_search_8_8 (&nm->in2out, &kv1, &value1) !=
1276 if (is_output_feature)
1278 if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1280 vnet_buffer (b1)->ip.reass.l4_src_port,
1281 vnet_buffer (b1)->ip.reass.l4_dst_port, thread_index,
1286 * Send DHCP packets to the ipv4 stack, or we won't
1287 * be able to use dhcp client on the outside interface
1290 (proto1 == NAT_PROTOCOL_UDP
1291 && (vnet_buffer (b1)->ip.reass.l4_dst_port ==
1292 clib_host_to_net_u16
1293 (UDP_DST_PORT_dhcp_to_server))
1294 && ip1->dst_address.as_u32 == 0xffffffff))
1299 if (PREDICT_FALSE (nat44_ei_not_translate (
1300 nm, node, rx_sw_if_index1, ip1, proto1, rx_fib_index1,
1305 next1 = slow_path (nm, b1, ip1, ip1->src_address,
1306 vnet_buffer (b1)->ip.reass.l4_src_port,
1307 rx_fib_index1, proto1, &s1, node, next1,
1309 if (PREDICT_FALSE (next1 == NAT44_EI_IN2OUT_NEXT_DROP))
1312 if (PREDICT_FALSE (!s1))
1317 next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1322 s1 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1323 nat_value_get_session_index (&value1));
1325 b1->flags |= VNET_BUFFER_F_IS_NATED;
1327 old_addr1 = ip1->src_address.as_u32;
1328 ip1->src_address = s1->out2in.addr;
1329 new_addr1 = ip1->src_address.as_u32;
1330 if (!is_output_feature)
1331 vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1333 sum1 = ip1->checksum;
1334 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1335 ip4_header_t, src_address /* changed member */ );
1336 ip1->checksum = ip_csum_fold (sum1);
1338 if (PREDICT_TRUE (proto1 == NAT_PROTOCOL_TCP))
1340 if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1342 old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1343 new_port1 = udp1->src_port = s1->out2in.port;
1344 sum1 = tcp1->checksum;
1345 sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1347 dst_address /* changed member */ );
1348 sum1 = ip_csum_update (sum1, old_port1, new_port1,
1349 ip4_header_t /* cheat */ ,
1350 length /* changed member */ );
1351 mss_clamping (nm->mss_clamping, tcp1, &sum1);
1352 tcp1->checksum = ip_csum_fold (sum1);
1354 vlib_increment_simple_counter (is_slow_path ?
1355 &nm->counters.slowpath.in2out.tcp :
1356 &nm->counters.fastpath.in2out.tcp,
1357 thread_index, cntr_sw_if_index1, 1);
1361 if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1363 udp1->src_port = s1->out2in.port;
1364 if (PREDICT_FALSE (udp1->checksum))
1366 old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1367 new_port1 = udp1->src_port;
1368 sum1 = udp1->checksum;
1369 sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t, dst_address /* changed member */
1372 ip_csum_update (sum1, old_port1, new_port1,
1373 ip4_header_t /* cheat */ ,
1374 length /* changed member */ );
1375 udp1->checksum = ip_csum_fold (sum1);
1378 vlib_increment_simple_counter (is_slow_path ?
1379 &nm->counters.slowpath.in2out.udp :
1380 &nm->counters.fastpath.in2out.udp,
1381 thread_index, cntr_sw_if_index1, 1);
1385 nat44_ei_session_update_counters (
1386 s1, now, vlib_buffer_length_in_chain (vm, b1), thread_index);
1387 /* Per-user LRU list maintenance */
1388 nat44_ei_session_update_lru (nm, s1, thread_index);
1391 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1392 && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1394 nat44_ei_in2out_trace_t *t =
1395 vlib_add_trace (vm, node, b1, sizeof (*t));
1396 t->sw_if_index = rx_sw_if_index1;
1397 t->next_index = next1;
1398 t->session_index = ~0;
1400 t->session_index = s1 - nm->per_thread_data[thread_index].sessions;
1403 if (next1 == NAT44_EI_IN2OUT_NEXT_DROP)
1405 vlib_increment_simple_counter (
1406 is_slow_path ? &nm->counters.slowpath.in2out.drops :
1407 &nm->counters.fastpath.in2out.drops,
1408 thread_index, cntr_sw_if_index1, 1);
1417 while (n_left_from > 0)
1421 u32 rx_sw_if_index0;
1422 u32 tx_sw_if_index0;
1423 u32 cntr_sw_if_index0;
1426 u32 new_addr0, old_addr0;
1427 u16 old_port0, new_port0;
1430 icmp46_header_t *icmp0;
1433 nat44_ei_session_t *s0 = 0;
1434 clib_bihash_kv_8_8_t kv0, value0;
1435 u32 iph_offset0 = 0;
1439 next0 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
1441 if (is_output_feature)
1442 iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1444 ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1447 udp0 = ip4_next_header (ip0);
1448 tcp0 = (tcp_header_t *) udp0;
1449 icmp0 = (icmp46_header_t *) udp0;
1451 rx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1452 tx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1454 is_output_feature ? tx_sw_if_index0 : rx_sw_if_index0;
1456 vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index0);
1458 if (PREDICT_FALSE (ip0->ttl == 1))
1460 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1461 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1462 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1464 next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1468 proto0 = ip_proto_to_nat_proto (ip0->protocol);
1470 /* Next configured feature, probably ip4-lookup */
1473 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1475 if (nat_in2out_sm_unknown_proto (nm, b0, ip0, rx_fib_index0))
1477 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1479 node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1481 vlib_increment_simple_counter (
1482 is_slow_path ? &nm->counters.slowpath.in2out.other :
1483 &nm->counters.fastpath.in2out.other,
1484 thread_index, cntr_sw_if_index0, 1);
1488 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1490 next0 = nat44_ei_icmp_in2out_slow_path (
1491 nm, b0, ip0, icmp0, rx_sw_if_index0, rx_fib_index0, node,
1492 next0, now, thread_index, &s0);
1493 vlib_increment_simple_counter (
1494 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1495 &nm->counters.fastpath.in2out.icmp,
1496 thread_index, cntr_sw_if_index0, 1);
1502 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1504 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1508 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1510 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1515 init_nat_k (&kv0, ip0->src_address,
1516 vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1519 if (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
1523 if (is_output_feature)
1525 if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1527 vnet_buffer (b0)->ip.reass.l4_src_port,
1528 vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1533 * Send DHCP packets to the ipv4 stack, or we won't
1534 * be able to use dhcp client on the outside interface
1537 (proto0 == NAT_PROTOCOL_UDP
1538 && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1539 clib_host_to_net_u16
1540 (UDP_DST_PORT_dhcp_to_server))
1541 && ip0->dst_address.as_u32 == 0xffffffff))
1546 if (PREDICT_FALSE (nat44_ei_not_translate (
1547 nm, node, rx_sw_if_index0, ip0, proto0, rx_fib_index0,
1552 next0 = slow_path (nm, b0, ip0, ip0->src_address,
1553 vnet_buffer (b0)->ip.reass.l4_src_port,
1554 rx_fib_index0, proto0, &s0, node, next0,
1557 if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
1560 if (PREDICT_FALSE (!s0))
1565 next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1570 s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1571 nat_value_get_session_index (&value0));
1573 b0->flags |= VNET_BUFFER_F_IS_NATED;
1575 old_addr0 = ip0->src_address.as_u32;
1576 ip0->src_address = s0->out2in.addr;
1577 new_addr0 = ip0->src_address.as_u32;
1578 if (!is_output_feature)
1579 vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1581 sum0 = ip0->checksum;
1582 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1583 ip4_header_t, src_address /* changed member */ );
1584 ip0->checksum = ip_csum_fold (sum0);
1586 if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1588 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1590 old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1591 new_port0 = udp0->src_port = s0->out2in.port;
1592 sum0 = tcp0->checksum;
1594 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1595 dst_address /* changed member */ );
1597 ip_csum_update (sum0, old_port0, new_port0,
1598 ip4_header_t /* cheat */ ,
1599 length /* changed member */ );
1600 mss_clamping (nm->mss_clamping, tcp0, &sum0);
1601 tcp0->checksum = ip_csum_fold (sum0);
1603 vlib_increment_simple_counter (is_slow_path ?
1604 &nm->counters.slowpath.in2out.tcp :
1605 &nm->counters.fastpath.in2out.tcp,
1606 thread_index, cntr_sw_if_index0, 1);
1610 if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1612 udp0->src_port = s0->out2in.port;
1613 if (PREDICT_FALSE (udp0->checksum))
1615 old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1616 new_port0 = udp0->src_port;
1617 sum0 = udp0->checksum;
1619 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1620 dst_address /* changed member */ );
1622 ip_csum_update (sum0, old_port0, new_port0,
1623 ip4_header_t /* cheat */ ,
1624 length /* changed member */ );
1625 udp0->checksum = ip_csum_fold (sum0);
1628 vlib_increment_simple_counter (is_slow_path ?
1629 &nm->counters.slowpath.in2out.udp :
1630 &nm->counters.fastpath.in2out.udp,
1631 thread_index, cntr_sw_if_index0, 1);
1635 nat44_ei_session_update_counters (
1636 s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1637 /* Per-user LRU list maintenance */
1638 nat44_ei_session_update_lru (nm, s0, thread_index);
1641 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1642 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1644 nat44_ei_in2out_trace_t *t =
1645 vlib_add_trace (vm, node, b0, sizeof (*t));
1646 t->is_slow_path = is_slow_path;
1647 t->sw_if_index = rx_sw_if_index0;
1648 t->next_index = next0;
1649 t->session_index = ~0;
1651 t->session_index = s0 - nm->per_thread_data[thread_index].sessions;
1654 if (next0 == NAT44_EI_IN2OUT_NEXT_DROP)
1656 vlib_increment_simple_counter (
1657 is_slow_path ? &nm->counters.slowpath.in2out.drops :
1658 &nm->counters.fastpath.in2out.drops,
1659 thread_index, cntr_sw_if_index0, 1);
1667 vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1669 return frame->n_vectors;
1672 VLIB_NODE_FN (nat44_ei_in2out_node)
1673 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1675 return nat44_ei_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */,
1679 VLIB_REGISTER_NODE (nat44_ei_in2out_node) = {
1680 .name = "nat44-ei-in2out",
1681 .vector_size = sizeof (u32),
1682 .format_trace = format_nat44_ei_in2out_trace,
1683 .type = VLIB_NODE_TYPE_INTERNAL,
1685 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1686 .error_strings = nat44_ei_in2out_error_strings,
1688 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1690 .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1692 /* edit / add dispositions here */
1694 [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1695 [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1696 [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
1697 [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1698 [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
1702 VLIB_NODE_FN (nat44_ei_in2out_output_node)
1703 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1705 return nat44_ei_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */,
1709 VLIB_REGISTER_NODE (nat44_ei_in2out_output_node) = {
1710 .name = "nat44-ei-in2out-output",
1711 .vector_size = sizeof (u32),
1712 .format_trace = format_nat44_ei_in2out_trace,
1713 .type = VLIB_NODE_TYPE_INTERNAL,
1715 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1716 .error_strings = nat44_ei_in2out_error_strings,
1718 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1720 .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1722 /* edit / add dispositions here */
1724 [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1725 [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "interface-output",
1726 [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-output-slowpath",
1727 [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1728 [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-interface-output",
1732 VLIB_NODE_FN (nat44_ei_in2out_slowpath_node)
1733 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1735 return nat44_ei_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */,
1739 VLIB_REGISTER_NODE (nat44_ei_in2out_slowpath_node) = {
1740 .name = "nat44-ei-in2out-slowpath",
1741 .vector_size = sizeof (u32),
1742 .format_trace = format_nat44_ei_in2out_trace,
1743 .type = VLIB_NODE_TYPE_INTERNAL,
1745 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1746 .error_strings = nat44_ei_in2out_error_strings,
1748 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1750 .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1752 /* edit / add dispositions here */
1754 [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1755 [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1756 [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
1757 [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1758 [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
1762 VLIB_NODE_FN (nat44_ei_in2out_output_slowpath_node)
1763 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1765 return nat44_ei_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */,
1769 VLIB_REGISTER_NODE (nat44_ei_in2out_output_slowpath_node) = {
1770 .name = "nat44-ei-in2out-output-slowpath",
1771 .vector_size = sizeof (u32),
1772 .format_trace = format_nat44_ei_in2out_trace,
1773 .type = VLIB_NODE_TYPE_INTERNAL,
1775 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1776 .error_strings = nat44_ei_in2out_error_strings,
1778 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1780 .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1782 /* edit / add dispositions here */
1784 [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1785 [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "interface-output",
1786 [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-output-slowpath",
1787 [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1788 [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-interface-output",
1792 VLIB_NODE_FN (nat44_ei_in2out_fast_node)
1793 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1795 u32 n_left_from, *from, *to_next;
1796 u32 thread_index = vm->thread_index;
1797 nat44_ei_in2out_next_t next_index;
1798 nat44_ei_main_t *nm = &nat44_ei_main;
1799 int is_hairpinning = 0;
1801 from = vlib_frame_vector_args (frame);
1802 n_left_from = frame->n_vectors;
1803 next_index = node->cached_next_index;
1805 while (n_left_from > 0)
1809 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1811 while (n_left_from > 0 && n_left_to_next > 0)
1819 u32 new_addr0, old_addr0;
1820 u16 old_port0, new_port0;
1823 icmp46_header_t *icmp0;
1826 ip4_address_t sm0_addr;
1829 u32 required_thread_index = thread_index;
1831 /* speculatively enqueue b0 to the current next frame */
1837 n_left_to_next -= 1;
1839 b0 = vlib_get_buffer (vm, bi0);
1840 next0 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
1842 ip0 = vlib_buffer_get_current (b0);
1843 udp0 = ip4_next_header (ip0);
1844 tcp0 = (tcp_header_t *) udp0;
1845 icmp0 = (icmp46_header_t *) udp0;
1847 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1849 ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1851 if (PREDICT_FALSE (ip0->ttl == 1))
1853 vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1854 icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1855 ICMP4_time_exceeded_ttl_exceeded_in_transit,
1857 next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1861 proto0 = ip_proto_to_nat_proto (ip0->protocol);
1863 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1866 if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1868 next0 = nat44_ei_icmp_in2out (b0, ip0, icmp0, sw_if_index0,
1869 rx_fib_index0, node, next0, ~0, 0);
1873 if (nat44_ei_static_mapping_match (
1874 ip0->src_address, udp0->src_port, rx_fib_index0, proto0,
1875 &sm0_addr, &sm0_port, &sm0_fib_index, 0, 0, 0))
1877 b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_NO_TRANSLATION];
1878 next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1882 new_addr0 = sm0_addr.as_u32;
1883 new_port0 = sm0_port;
1884 vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0_fib_index;
1885 old_addr0 = ip0->src_address.as_u32;
1886 ip0->src_address.as_u32 = new_addr0;
1888 sum0 = ip0->checksum;
1889 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1891 src_address /* changed member */ );
1892 ip0->checksum = ip_csum_fold (sum0);
1894 if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1896 old_port0 = udp0->src_port;
1897 udp0->src_port = new_port0;
1899 if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1901 sum0 = tcp0->checksum;
1902 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1904 dst_address /* changed member */ );
1905 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1906 ip4_header_t /* cheat */ ,
1907 length /* changed member */ );
1908 mss_clamping (nm->mss_clamping, tcp0, &sum0);
1909 tcp0->checksum = ip_csum_fold (sum0);
1911 else if (udp0->checksum)
1913 sum0 = udp0->checksum;
1914 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1916 dst_address /* changed member */ );
1917 sum0 = ip_csum_update (sum0, old_port0, new_port0,
1918 ip4_header_t /* cheat */ ,
1919 length /* changed member */ );
1920 udp0->checksum = ip_csum_fold (sum0);
1925 if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1927 sum0 = tcp0->checksum;
1928 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1930 dst_address /* changed member */ );
1931 mss_clamping (nm->mss_clamping, tcp0, &sum0);
1932 tcp0->checksum = ip_csum_fold (sum0);
1934 else if (udp0->checksum)
1936 sum0 = udp0->checksum;
1937 sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1939 dst_address /* changed member */ );
1940 udp0->checksum = ip_csum_fold (sum0);
1945 is_hairpinning = nat44_ei_hairpinning (
1946 vm, node, nm, thread_index, b0, ip0, udp0, tcp0, proto0,
1947 0 /* do_trace */, &required_thread_index);
1949 if (thread_index != required_thread_index)
1951 vnet_buffer (b0)->snat.required_thread_index =
1952 required_thread_index;
1953 next0 = NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF;
1957 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1958 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1960 nat44_ei_in2out_trace_t *t =
1961 vlib_add_trace (vm, node, b0, sizeof (*t));
1962 t->sw_if_index = sw_if_index0;
1963 t->next_index = next0;
1964 t->is_hairpinning = is_hairpinning;
1967 if (next0 != NAT44_EI_IN2OUT_NEXT_DROP)
1970 vlib_increment_simple_counter (
1971 &nm->counters.fastpath.in2out.other, sw_if_index0,
1972 vm->thread_index, 1);
1975 /* verify speculative enqueue, maybe switch current next frame */
1976 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1977 to_next, n_left_to_next,
1981 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1984 return frame->n_vectors;
1987 VLIB_REGISTER_NODE (nat44_ei_in2out_fast_node) = {
1988 .name = "nat44-ei-in2out-fast",
1989 .vector_size = sizeof (u32),
1990 .format_trace = format_nat44_ei_in2out_fast_trace,
1991 .type = VLIB_NODE_TYPE_INTERNAL,
1993 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1994 .error_strings = nat44_ei_in2out_error_strings,
1996 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1998 .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
2000 /* edit / add dispositions here */
2002 [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
2003 [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2004 [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
2005 [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2006 [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2010 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_handoff_ip4_lookup_node)
2011 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2013 return nat44_ei_hairpinning_handoff_fn_inline (
2015 nat44_ei_main.in2out_hairpinning_finish_ip4_lookup_node_fq_index);
2018 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_handoff_ip4_lookup_node) = {
2019 .name = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2020 .vector_size = sizeof (u32),
2021 .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2022 .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2023 .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2032 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_handoff_interface_output_node)
2033 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2035 return nat44_ei_hairpinning_handoff_fn_inline (
2037 nat44_ei_main.in2out_hairpinning_finish_interface_output_node_fq_index);
2040 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_handoff_interface_output_node) = {
2041 .name = "nat44-ei-in2out-hairpinning-handoff-interface-output",
2042 .vector_size = sizeof (u32),
2043 .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2044 .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2045 .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2054 static_always_inline int
2055 nat44_ei_in2out_hairpinning_finish_inline (vlib_main_t *vm,
2056 vlib_node_runtime_t *node,
2057 vlib_frame_t *frame)
2059 u32 n_left_from, *from, *to_next;
2060 u32 thread_index = vm->thread_index;
2061 nat44_ei_in2out_next_t next_index;
2062 nat44_ei_main_t *nm = &nat44_ei_main;
2063 int is_hairpinning = 0;
2065 from = vlib_frame_vector_args (frame);
2066 n_left_from = frame->n_vectors;
2067 next_index = node->cached_next_index;
2069 while (n_left_from > 0)
2073 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2075 while (n_left_from > 0 && n_left_to_next > 0)
2084 icmp46_header_t *icmp0;
2086 u32 required_thread_index = thread_index;
2088 /* speculatively enqueue b0 to the current next frame */
2094 n_left_to_next -= 1;
2096 b0 = vlib_get_buffer (vm, bi0);
2097 next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP;
2099 ip0 = vlib_buffer_get_current (b0);
2100 udp0 = ip4_next_header (ip0);
2101 tcp0 = (tcp_header_t *) udp0;
2102 icmp0 = (icmp46_header_t *) udp0;
2104 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2105 proto0 = ip_proto_to_nat_proto (ip0->protocol);
2109 case NAT_PROTOCOL_TCP:
2111 case NAT_PROTOCOL_UDP:
2112 is_hairpinning = nat44_ei_hairpinning (
2113 vm, node, nm, thread_index, b0, ip0, udp0, tcp0, proto0,
2114 0 /* do_trace */, &required_thread_index);
2116 case NAT_PROTOCOL_ICMP:
2117 is_hairpinning = (0 == nat44_ei_icmp_hairpinning (
2118 nm, b0, thread_index, ip0, icmp0,
2119 &required_thread_index));
2121 case NAT_PROTOCOL_OTHER:
2122 // this should never happen
2123 next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2127 if (thread_index != required_thread_index)
2129 // but we already did a handoff ...
2130 next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2133 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
2134 (b0->flags & VLIB_BUFFER_IS_TRACED)))
2136 nat44_ei_in2out_trace_t *t =
2137 vlib_add_trace (vm, node, b0, sizeof (*t));
2138 t->sw_if_index = sw_if_index0;
2139 t->next_index = next0;
2140 t->is_hairpinning = is_hairpinning;
2143 if (next0 != NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP)
2145 vlib_increment_simple_counter (
2146 &nm->counters.fastpath.in2out.other, sw_if_index0,
2147 vm->thread_index, 1);
2150 /* verify speculative enqueue, maybe switch current next frame */
2151 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2152 n_left_to_next, bi0, next0);
2155 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2158 return frame->n_vectors;
2161 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_finish_ip4_lookup_node)
2162 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2164 return nat44_ei_in2out_hairpinning_finish_inline (vm, node, frame);
2167 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_finish_ip4_lookup_node) = {
2168 .name = "nat44-ei-in2out-hairpinning-finish-ip4-lookup",
2169 .vector_size = sizeof (u32),
2170 .format_trace = format_nat44_ei_in2out_fast_trace,
2171 .type = VLIB_NODE_TYPE_INTERNAL,
2173 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2174 .error_strings = nat44_ei_in2out_error_strings,
2176 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2178 .n_next_nodes = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2180 /* edit / add dispositions here */
2182 [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2183 [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "ip4-lookup",
2187 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_finish_interface_output_node)
2188 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2190 return nat44_ei_in2out_hairpinning_finish_inline (vm, node, frame);
2193 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_finish_interface_output_node) = {
2194 .name = "nat44-ei-in2out-hairpinning-finish-interface-output",
2195 .vector_size = sizeof (u32),
2196 .format_trace = format_nat44_ei_in2out_fast_trace,
2197 .type = VLIB_NODE_TYPE_INTERNAL,
2199 .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2200 .error_strings = nat44_ei_in2out_error_strings,
2202 .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2204 .n_next_nodes = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2206 /* edit / add dispositions here */
2208 [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2209 [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "interface-output",
2214 * fd.io coding-style-patch-verification: ON
2217 * eval: (c-set-style "gnu")