src/plugins/nat/nat44-ei/nat44_ei_in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 EI inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21
  22 #include <vnet/vnet.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/udp/udp_local.h>
  26 #include <vnet/fib/ip4_fib.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/ipfix_logging.h>
  34 #include <nat/lib/nat_inlines.h>
  35 #include <nat/nat44-ei/nat44_ei_inlines.h>
  36 #include <nat/nat44-ei/nat44_ei.h>
  37
  38 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
  39
  40 #define foreach_nat44_ei_in2out_error                                         \
  41   _ (UNSUPPORTED_PROTOCOL, "unsupported protocol")                            \
  42   _ (OUT_OF_PORTS, "out of ports")                                            \
  43   _ (BAD_OUTSIDE_FIB, "outside VRF ID not found")                             \
  44   _ (BAD_ICMP_TYPE, "unsupported ICMP type")                                  \
  45   _ (NO_TRANSLATION, "no translation")                                        \
  46   _ (MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")                      \
  47   _ (CANNOT_CREATE_USER, "cannot create NAT user")
  48
  49 #define foreach_nat44_ei_hairpinning_handoff_error                            \
  50   _ (CONGESTION_DROP, "congestion drop")
  51
  52 typedef enum
  53 {
  54 #define _(sym, str) NAT44_EI_IN2OUT_ERROR_##sym,
  55   foreach_nat44_ei_in2out_error
  56 #undef _
  57     NAT44_EI_IN2OUT_N_ERROR,
  58 } nat44_ei_in2out_error_t;
  59
  60 static char *nat44_ei_in2out_error_strings[] = {
  61 #define _(sym,string) string,
  62   foreach_nat44_ei_in2out_error
  63 #undef _
  64 };
  65
  66 typedef enum
  67 {
  68 #define _(sym, str) NAT44_EI_HAIRPINNING_HANDOFF_ERROR_##sym,
  69   foreach_nat44_ei_hairpinning_handoff_error
  70 #undef _
  71     NAT44_EI_HAIRPINNING_HANDOFF_N_ERROR,
  72 } nat44_ei_hairpinning_handoff_error_t;
  73
  74 static char *nat44_ei_hairpinning_handoff_error_strings[] = {
  75 #define _(sym, string) string,
  76   foreach_nat44_ei_hairpinning_handoff_error
  77 #undef _
  78 };
  79
  80 typedef enum
  81 {
  82   NAT44_EI_IN2OUT_NEXT_LOOKUP,
  83   NAT44_EI_IN2OUT_NEXT_DROP,
  84   NAT44_EI_IN2OUT_NEXT_ICMP_ERROR,
  85   NAT44_EI_IN2OUT_NEXT_SLOW_PATH,
  86   NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF,
  87   NAT44_EI_IN2OUT_N_NEXT,
  88 } nat44_ei_in2out_next_t;
  89
  90 typedef enum
  91 {
  92   NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP,
  93   NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP,
  94   NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
  95 } nat44_ei_in2out_hairpinnig_finish_next_t;
  96
  97 typedef enum
  98 {
  99   NAT44_EI_HAIRPIN_NEXT_LOOKUP,
 100   NAT44_EI_HAIRPIN_NEXT_DROP,
 101   NAT44_EI_HAIRPIN_NEXT_HANDOFF,
 102   NAT44_EI_HAIRPIN_N_NEXT,
 103 } nat44_ei_hairpin_next_t;
 104
 105 typedef struct
 106 {
 107   u32 sw_if_index;
 108   u32 next_index;
 109   u32 session_index;
 110   u32 is_slow_path;
 111   u32 is_hairpinning;
 112 } nat44_ei_in2out_trace_t;
 113
 114 typedef struct
 115 {
 116   ip4_address_t addr;
 117   u16 port;
 118   u32 fib_index;
 119   u32 session_index;
 120 } nat44_ei_hairpin_trace_t;
 121
 122 typedef struct
 123 {
 124   u32 next_worker_index;
 125 } nat44_ei_hairpinning_handoff_trace_t;
 126
 127 static u8 *
 128 format_nat44_ei_in2out_trace (u8 *s, va_list *args)
 129 {
 130   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 131   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 132   nat44_ei_in2out_trace_t *t = va_arg (*args, nat44_ei_in2out_trace_t *);
 133   char *tag;
 134   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
 135   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
 136               t->sw_if_index, t->next_index, t->session_index);
 137   if (t->is_hairpinning)
 138     s = format (s, ", with-hairpinning");
 139   return s;
 140 }
 141
 142 static u8 *
 143 format_nat44_ei_in2out_fast_trace (u8 *s, va_list *args)
 144 {
 145   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 146   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 147   nat44_ei_in2out_trace_t *t = va_arg (*args, nat44_ei_in2out_trace_t *);
 148   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
 149               t->sw_if_index, t->next_index);
 150   return s;
 151 }
 152
 153 static u8 *
 154 format_nat44_ei_hairpin_trace (u8 *s, va_list *args)
 155 {
 156   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 157   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 158   nat44_ei_hairpin_trace_t *t = va_arg (*args, nat44_ei_hairpin_trace_t *);
 159
 160   s = format (s, "new dst addr %U port %u fib-index %u", format_ip4_address,
 161               &t->addr, clib_net_to_host_u16 (t->port), t->fib_index);
 162   if (~0 == t->session_index)
 163     {
 164       s = format (s, " is-static-mapping");
 165     }
 166   else
 167     {
 168       s = format (s, " session-index %u", t->session_index);
 169     }
 170
 171   return s;
 172 }
 173
 174 static u8 *
 175 format_nat44_ei_hairpinning_handoff_trace (u8 *s, va_list *args)
 176 {
 177   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 178   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 179   nat44_ei_hairpinning_handoff_trace_t *t =
 180     va_arg (*args, nat44_ei_hairpinning_handoff_trace_t *);
 181
 182   s = format (s, "nat44-ei-hairpinning-handoff: next-worker %d",
 183               t->next_worker_index);
 184
 185   return s;
 186 }
 187
 188 static_always_inline int
 189 nat44_ei_not_translate_fast (vlib_node_runtime_t *node, u32 sw_if_index0,
 190                              ip4_header_t *ip0, u32 proto0, u32 rx_fib_index0)
 191 {
 192   nat44_ei_main_t *nm = &nat44_ei_main;
 193
 194   if (nm->out2in_dpo)
 195     return 0;
 196
 197   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 198   nat44_ei_outside_fib_t *outside_fib;
 199   fib_prefix_t pfx = {
 200     .fp_proto = FIB_PROTOCOL_IP4,
 201     .fp_len = 32,
 202     .fp_addr = {
 203                 .ip4.as_u32 = ip0->dst_address.as_u32,
 204                 }
 205     ,
 206   };
 207
 208   /* Don't NAT packet aimed at the intfc address */
 209   if (PREDICT_FALSE (nat44_ei_is_interface_addr (
 210         nm->ip4_main, node, sw_if_index0, ip0->dst_address.as_u32)))
 211     return 1;
 212
 213   fei = fib_table_lookup (rx_fib_index0, &pfx);
 214   if (FIB_NODE_INDEX_INVALID != fei)
 215     {
 216       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 217       if (sw_if_index == ~0)
 218         {
 219           vec_foreach (outside_fib, nm->outside_fibs)
 220             {
 221               fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 222               if (FIB_NODE_INDEX_INVALID != fei)
 223                 {
 224                   sw_if_index = fib_entry_get_resolving_interface (fei);
 225                   if (sw_if_index != ~0)
 226                     break;
 227                 }
 228             }
 229         }
 230       if (sw_if_index == ~0)
 231         return 1;
 232
 233       nat44_ei_interface_t *i;
 234       pool_foreach (i, nm->interfaces)
 235         {
 236           /* NAT packet aimed at outside interface */
 237           if ((nat44_ei_interface_is_outside (i)) &&
 238               (sw_if_index == i->sw_if_index))
 239             return 0;
 240         }
 241     }
 242
 243   return 1;
 244 }
 245
 246 static_always_inline int
 247 nat44_ei_not_translate (nat44_ei_main_t *nm, vlib_node_runtime_t *node,
 248                         u32 sw_if_index0, ip4_header_t *ip0, u32 proto0,
 249                         u32 rx_fib_index0, u32 thread_index)
 250 {
 251   udp_header_t *udp0 = ip4_next_header (ip0);
 252   clib_bihash_kv_8_8_t kv0, value0;
 253
 254   init_nat_k (&kv0, ip0->dst_address, udp0->dst_port, nm->outside_fib_index,
 255               proto0);
 256
 257   /* NAT packet aimed at external address if */
 258   /* has active sessions */
 259   if (clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
 260     {
 261       /* or is static mappings */
 262       ip4_address_t placeholder_addr;
 263       u16 placeholder_port;
 264       u32 placeholder_fib_index;
 265       if (!nat44_ei_static_mapping_match (ip0->dst_address, udp0->dst_port,
 266                                           nm->outside_fib_index, proto0,
 267                                           &placeholder_addr, &placeholder_port,
 268                                           &placeholder_fib_index, 1, 0, 0))
 269         return 0;
 270     }
 271   else
 272     return 0;
 273
 274   if (nm->forwarding_enabled)
 275     return 1;
 276
 277   return nat44_ei_not_translate_fast (node, sw_if_index0, ip0, proto0,
 278                                       rx_fib_index0);
 279 }
 280
 281 static_always_inline int
 282 nat44_ei_not_translate_output_feature (nat44_ei_main_t *nm, ip4_header_t *ip0,
 283                                        u32 proto0, u16 src_port, u16 dst_port,
 284                                        u32 thread_index, u32 sw_if_index)
 285 {
 286   clib_bihash_kv_8_8_t kv0, value0;
 287   nat44_ei_interface_t *i;
 288
 289   /* src NAT check */
 290   init_nat_k (&kv0, ip0->src_address, src_port,
 291               ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
 292
 293   if (!clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
 294     return 1;
 295
 296   /* dst NAT check */
 297   init_nat_k (&kv0, ip0->dst_address, dst_port,
 298               ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
 299   if (!clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
 300     {
 301       /* hairpinning */
 302       pool_foreach (i, nm->output_feature_interfaces)
 303         {
 304           if ((nat44_ei_interface_is_inside (i)) &&
 305               (sw_if_index == i->sw_if_index))
 306             return 0;
 307         }
 308       return 1;
 309     }
 310
 311   return 0;
 312 }
 313
 314 #ifndef CLIB_MARCH_VARIANT
 315 int
 316 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 317 {
 318   nat44_ei_main_t *nm = &nat44_ei_main;
 319   nat44_ei_is_idle_session_ctx_t *ctx = arg;
 320   nat44_ei_session_t *s;
 321   u64 sess_timeout_time;
 322   nat44_ei_main_per_thread_data_t *tnm =
 323     vec_elt_at_index (nm->per_thread_data, ctx->thread_index);
 324   clib_bihash_kv_8_8_t s_kv;
 325
 326   if (ctx->thread_index != nat_value_get_thread_index (kv))
 327     {
 328       return 0;
 329     }
 330
 331   s = pool_elt_at_index (tnm->sessions, nat_value_get_session_index (kv));
 332   sess_timeout_time = s->last_heard + (f64) nat_session_get_timeout (
 333                                         &nm->timeouts, s->nat_proto, s->state);
 334   if (ctx->now >= sess_timeout_time)
 335     {
 336       init_nat_o2i_k (&s_kv, s);
 337       if (clib_bihash_add_del_8_8 (&nm->out2in, &s_kv, 0))
 338         nat_elog_warn (nm, "out2in key del failed");
 339
 340       nat_ipfix_logging_nat44_ses_delete (
 341         ctx->thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
 342         nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
 343         s->in2out.fib_index);
 344
 345       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 346                                &s->in2out.addr, s->in2out.port,
 347                                &s->out2in.addr, s->out2in.port, s->nat_proto);
 348
 349       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 350                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 351                    ctx->thread_index);
 352
 353       if (!nat44_ei_is_session_static (s))
 354         nat44_ei_free_outside_address_and_port (
 355           nm->addresses, ctx->thread_index, &s->out2in.addr, s->out2in.port,
 356           s->nat_proto);
 357
 358       nat44_ei_delete_session (nm, s, ctx->thread_index);
 359       return 1;
 360     }
 361
 362   return 0;
 363 }
 364 #endif
 365
 366 static u32
 367 slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0, ip4_header_t *ip0,
 368            ip4_address_t i2o_addr, u16 i2o_port, u32 rx_fib_index0,
 369            nat_protocol_t nat_proto, nat44_ei_session_t **sessionp,
 370            vlib_node_runtime_t *node, u32 next0, u32 thread_index, f64 now)
 371 {
 372   nat44_ei_user_t *u;
 373   nat44_ei_session_t *s = 0;
 374   clib_bihash_kv_8_8_t kv0;
 375   u8 is_sm = 0;
 376   nat44_ei_outside_fib_t *outside_fib;
 377   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 378   u8 identity_nat;
 379   fib_prefix_t pfx = {
 380     .fp_proto = FIB_PROTOCOL_IP4,
 381     .fp_len = 32,
 382     .fp_addr = {
 383                 .ip4.as_u32 = ip0->dst_address.as_u32,
 384                 },
 385   };
 386   nat44_ei_is_idle_session_ctx_t ctx0;
 387   ip4_address_t sm_addr;
 388   u16 sm_port;
 389   u32 sm_fib_index;
 390
 391   if (PREDICT_FALSE (nat44_ei_maximum_sessions_exceeded (nm, thread_index)))
 392     {
 393       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 394       nat_ipfix_logging_max_sessions (thread_index,
 395                                       nm->max_translations_per_thread);
 396       nat_elog_notice (nm, "maximum sessions exceeded");
 397       return NAT44_EI_IN2OUT_NEXT_DROP;
 398     }
 399
 400   /* First try to match static mapping by local address and port */
 401   if (nat44_ei_static_mapping_match (i2o_addr, i2o_port, rx_fib_index0,
 402                                      nat_proto, &sm_addr, &sm_port,
 403                                      &sm_fib_index, 0, 0, &identity_nat))
 404     {
 405       /* Try to create dynamic translation */
 406       if (nm->alloc_addr_and_port (
 407             nm->addresses, rx_fib_index0, thread_index, nat_proto,
 408             ip0->src_address, &sm_addr, &sm_port, nm->port_per_thread,
 409             nm->per_thread_data[thread_index].snat_thread_index))
 410         {
 411           b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_OUT_OF_PORTS];
 412           return NAT44_EI_IN2OUT_NEXT_DROP;
 413         }
 414     }
 415   else
 416     {
 417       if (PREDICT_FALSE (identity_nat))
 418         {
 419           *sessionp = s;
 420           return next0;
 421         }
 422
 423       is_sm = 1;
 424     }
 425
 426   u = nat44_ei_user_get_or_create (nm, &ip0->src_address, rx_fib_index0,
 427                                    thread_index);
 428   if (!u)
 429     {
 430       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_CANNOT_CREATE_USER];
 431       return NAT44_EI_IN2OUT_NEXT_DROP;
 432     }
 433
 434   s = nat44_ei_session_alloc_or_recycle (nm, u, thread_index, now);
 435   if (!s)
 436     {
 437       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
 438       nat_elog_warn (nm, "create NAT session failed");
 439       return NAT44_EI_IN2OUT_NEXT_DROP;
 440     }
 441
 442   if (is_sm)
 443     s->flags |= NAT44_EI_SESSION_FLAG_STATIC_MAPPING;
 444   nat44_ei_user_session_increment (nm, u, is_sm);
 445   s->in2out.addr = i2o_addr;
 446   s->in2out.port = i2o_port;
 447   s->in2out.fib_index = rx_fib_index0;
 448   s->nat_proto = nat_proto;
 449   s->out2in.addr = sm_addr;
 450   s->out2in.port = sm_port;
 451   s->out2in.fib_index = nm->outside_fib_index;
 452   switch (vec_len (nm->outside_fibs))
 453     {
 454     case 0:
 455       s->out2in.fib_index = nm->outside_fib_index;
 456       break;
 457     case 1:
 458       s->out2in.fib_index = nm->outside_fibs[0].fib_index;
 459       break;
 460     default:
 461       vec_foreach (outside_fib, nm->outside_fibs)
 462         {
 463           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 464           if (FIB_NODE_INDEX_INVALID != fei)
 465             {
 466               if (fib_entry_get_resolving_interface (fei) != ~0)
 467                 {
 468                   s->out2in.fib_index = outside_fib->fib_index;
 469                   break;
 470                 }
 471             }
 472         }
 473       break;
 474     }
 475   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 476   s->ext_host_port = vnet_buffer (b0)->ip.reass.l4_dst_port;
 477   *sessionp = s;
 478
 479   /* Add to translation hashes */
 480   ctx0.now = now;
 481   ctx0.thread_index = thread_index;
 482   init_nat_i2o_kv (&kv0, s, thread_index,
 483                    s - nm->per_thread_data[thread_index].sessions);
 484   if (clib_bihash_add_or_overwrite_stale_8_8 (
 485         &nm->in2out, &kv0, nat44_i2o_is_idle_session_cb, &ctx0))
 486     nat_elog_notice (nm, "in2out key add failed");
 487
 488   init_nat_o2i_kv (&kv0, s, thread_index,
 489                    s - nm->per_thread_data[thread_index].sessions);
 490   if (clib_bihash_add_or_overwrite_stale_8_8 (
 491         &nm->out2in, &kv0, nat44_o2i_is_idle_session_cb, &ctx0))
 492     nat_elog_notice (nm, "out2in key add failed");
 493
 494   /* log NAT event */
 495   nat_ipfix_logging_nat44_ses_create (
 496     thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32,
 497     nat_proto_to_ip_proto (s->nat_proto), s->in2out.port, s->out2in.port,
 498     s->in2out.fib_index);
 499
 500   nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index, &s->in2out.addr,
 501                            s->in2out.port, &s->out2in.addr, s->out2in.port,
 502                            s->nat_proto);
 503
 504   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 505                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 506                &s->ext_host_nat_addr, s->ext_host_nat_port, s->nat_proto,
 507                s->in2out.fib_index, s->flags, thread_index, 0);
 508
 509   return next0;
 510 }
 511
 512 static_always_inline nat44_ei_in2out_error_t
 513 icmp_get_key (vlib_buffer_t *b, ip4_header_t *ip0, ip4_address_t *addr,
 514               u16 *port, nat_protocol_t *nat_proto)
 515 {
 516   icmp46_header_t *icmp0;
 517   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 518   ip4_header_t *inner_ip0 = 0;
 519   void *l4_header = 0;
 520   icmp46_header_t *inner_icmp0;
 521
 522   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 523   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 524
 525   if (!icmp_type_is_error_message
 526       (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 527     {
 528       *nat_proto = NAT_PROTOCOL_ICMP;
 529       *addr = ip0->src_address;
 530       *port = vnet_buffer (b)->ip.reass.l4_src_port;
 531     }
 532   else
 533     {
 534       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 535       l4_header = ip4_next_header (inner_ip0);
 536       *nat_proto = ip_proto_to_nat_proto (inner_ip0->protocol);
 537       *addr = inner_ip0->dst_address;
 538       switch (*nat_proto)
 539         {
 540         case NAT_PROTOCOL_ICMP:
 541           inner_icmp0 = (icmp46_header_t *) l4_header;
 542           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 543           *port = inner_echo0->identifier;
 544           break;
 545         case NAT_PROTOCOL_UDP:
 546         case NAT_PROTOCOL_TCP:
 547           *port = ((tcp_udp_header_t *) l4_header)->dst_port;
 548           break;
 549         default:
 550           return NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 551         }
 552     }
 553   return -1;                    /* success */
 554 }
 555
 556 static_always_inline u32
 557 nat44_ei_icmp_match_in2out_slow (vlib_node_runtime_t *node, u32 thread_index,
 558                                  vlib_buffer_t *b0, ip4_header_t *ip0,
 559                                  ip4_address_t *addr, u16 *port,
 560                                  u32 *fib_index, nat_protocol_t *proto,
 561                                  nat44_ei_session_t **p_s0, u8 *dont_translate)
 562 {
 563   nat44_ei_main_t *nm = &nat44_ei_main;
 564   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
 565   u32 sw_if_index0;
 566   nat44_ei_session_t *s0 = 0;
 567   clib_bihash_kv_8_8_t kv0, value0;
 568   u32 next0 = ~0;
 569   int err;
 570   vlib_main_t *vm = vlib_get_main ();
 571   *dont_translate = 0;
 572
 573   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 574   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 575
 576   err = icmp_get_key (b0, ip0, addr, port, proto);
 577   if (err != -1)
 578     {
 579       b0->error = node->errors[err];
 580       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 581       goto out;
 582     }
 583
 584   init_nat_k (&kv0, *addr, *port, *fib_index, *proto);
 585   if (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
 586     {
 587       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 588         {
 589           if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
 590                 nm, ip0, *proto, *port, *port, thread_index, sw_if_index0)))
 591             {
 592               *dont_translate = 1;
 593               goto out;
 594             }
 595         }
 596       else
 597         {
 598           if (PREDICT_FALSE (nat44_ei_not_translate (
 599                 nm, node, sw_if_index0, ip0, NAT_PROTOCOL_ICMP, *fib_index,
 600                 thread_index)))
 601             {
 602               *dont_translate = 1;
 603               goto out;
 604             }
 605         }
 606
 607       if (PREDICT_FALSE
 608           (icmp_type_is_error_message
 609            (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags)))
 610         {
 611           b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
 612           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 613           goto out;
 614         }
 615
 616       next0 = slow_path (nm, b0, ip0, *addr, *port, *fib_index, *proto, &s0,
 617                          node, next0, thread_index, vlib_time_now (vm));
 618
 619       if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
 620         goto out;
 621
 622       if (!s0)
 623         {
 624           *dont_translate = 1;
 625           goto out;
 626         }
 627     }
 628   else
 629     {
 630       if (PREDICT_FALSE
 631           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 632            ICMP4_echo_request
 633            && vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 634            ICMP4_echo_reply
 635            && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 636                                            reass.icmp_type_or_tcp_flags)))
 637         {
 638           b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
 639           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 640           goto out;
 641         }
 642
 643       s0 = pool_elt_at_index (tnm->sessions,
 644                               nat_value_get_session_index (&value0));
 645     }
 646
 647 out:
 648   if (s0)
 649     {
 650       *addr = s0->out2in.addr;
 651       *port = s0->out2in.port;
 652       *fib_index = s0->out2in.fib_index;
 653     }
 654   if (p_s0)
 655     *p_s0 = s0;
 656   return next0;
 657 }
 658
 659 static_always_inline u32
 660 nat44_ei_icmp_match_in2out_fast (vlib_node_runtime_t *node, u32 thread_index,
 661                                  vlib_buffer_t *b0, ip4_header_t *ip0,
 662                                  ip4_address_t *addr, u16 *port,
 663                                  u32 *fib_index, nat_protocol_t *proto,
 664                                  nat44_ei_session_t **s0, u8 *dont_translate)
 665 {
 666   u32 sw_if_index0;
 667   u8 is_addr_only;
 668   u32 next0 = ~0;
 669   int err;
 670   *dont_translate = 0;
 671
 672   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 673   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 674
 675   err = icmp_get_key (b0, ip0, addr, port, proto);
 676   if (err != -1)
 677     {
 678       b0->error = node->errors[err];
 679       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 680       goto out;
 681     }
 682
 683   ip4_address_t sm_addr;
 684   u16 sm_port;
 685   u32 sm_fib_index;
 686
 687   if (nat44_ei_static_mapping_match (*addr, *port, *fib_index, *proto,
 688                                      &sm_addr, &sm_port, &sm_fib_index, 0,
 689                                      &is_addr_only, 0))
 690     {
 691       if (PREDICT_FALSE (nat44_ei_not_translate_fast (
 692             node, sw_if_index0, ip0, IP_PROTOCOL_ICMP, *fib_index)))
 693         {
 694           *dont_translate = 1;
 695           goto out;
 696         }
 697
 698       if (icmp_type_is_error_message
 699           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 700         {
 701           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 702           goto out;
 703         }
 704
 705       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_NO_TRANSLATION];
 706       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 707       goto out;
 708     }
 709
 710   if (PREDICT_FALSE
 711       (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags != ICMP4_echo_request
 712        && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 713            ICMP4_echo_reply || !is_addr_only)
 714        && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 715                                        reass.icmp_type_or_tcp_flags)))
 716     {
 717       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
 718       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 719       goto out;
 720     }
 721
 722 out:
 723   return next0;
 724 }
 725
 726 static_always_inline u32
 727 nat44_ei_icmp_hairpinning (nat44_ei_main_t *nm, vlib_buffer_t *b0,
 728                            u32 thread_index, ip4_header_t *ip0,
 729                            icmp46_header_t *icmp0, u32 *required_thread_index)
 730 {
 731   clib_bihash_kv_8_8_t kv0, value0;
 732   u32 old_dst_addr0, new_dst_addr0;
 733   u32 old_addr0, new_addr0;
 734   u16 old_port0, new_port0;
 735   u16 old_checksum0, new_checksum0;
 736   u32 si, ti = 0;
 737   ip_csum_t sum0;
 738   nat44_ei_session_t *s0;
 739   nat44_ei_static_mapping_t *m0;
 740
 741   if (icmp_type_is_error_message (
 742         vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 743     {
 744       ip4_header_t *inner_ip0 = 0;
 745       tcp_udp_header_t *l4_header = 0;
 746
 747       inner_ip0 = (ip4_header_t *) ((icmp_echo_header_t *) (icmp0 + 1) + 1);
 748       l4_header = ip4_next_header (inner_ip0);
 749       u32 protocol = ip_proto_to_nat_proto (inner_ip0->protocol);
 750
 751       if (protocol != NAT_PROTOCOL_TCP && protocol != NAT_PROTOCOL_UDP)
 752         return 1;
 753
 754       init_nat_k (&kv0, ip0->dst_address, l4_header->src_port,
 755                   nm->outside_fib_index, protocol);
 756       if (clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
 757         return 1;
 758       ti = nat_value_get_thread_index (&value0);
 759       if (ti != thread_index)
 760         {
 761           *required_thread_index = ti;
 762           return 1;
 763         }
 764       si = nat_value_get_session_index (&value0);
 765       s0 = pool_elt_at_index (nm->per_thread_data[ti].sessions, si);
 766       new_dst_addr0 = s0->in2out.addr.as_u32;
 767       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 768
 769       /* update inner source IP address */
 770       old_addr0 = inner_ip0->src_address.as_u32;
 771       inner_ip0->src_address.as_u32 = new_dst_addr0;
 772       new_addr0 = inner_ip0->src_address.as_u32;
 773       sum0 = icmp0->checksum;
 774       sum0 =
 775         ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, src_address);
 776       icmp0->checksum = ip_csum_fold (sum0);
 777
 778       /* update inner IP header checksum */
 779       old_checksum0 = inner_ip0->checksum;
 780       sum0 = inner_ip0->checksum;
 781       sum0 =
 782         ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, src_address);
 783       inner_ip0->checksum = ip_csum_fold (sum0);
 784       new_checksum0 = inner_ip0->checksum;
 785       sum0 = icmp0->checksum;
 786       sum0 = ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 787                              checksum);
 788       icmp0->checksum = ip_csum_fold (sum0);
 789
 790       /* update inner source port */
 791       old_port0 = l4_header->src_port;
 792       l4_header->src_port = s0->in2out.port;
 793       new_port0 = l4_header->src_port;
 794       sum0 = icmp0->checksum;
 795       sum0 = ip_csum_update (sum0, old_port0, new_port0, tcp_udp_header_t,
 796                              src_port);
 797       icmp0->checksum = ip_csum_fold (sum0);
 798     }
 799   else
 800     {
 801       init_nat_k (&kv0, ip0->dst_address, 0, nm->outside_fib_index, 0);
 802       if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv0,
 803                                   &value0))
 804         {
 805           icmp_echo_header_t *echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 806           u16 icmp_id0 = echo0->identifier;
 807           init_nat_k (&kv0, ip0->dst_address, icmp_id0, nm->outside_fib_index,
 808                       NAT_PROTOCOL_ICMP);
 809           int rv = clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0);
 810           if (!rv)
 811             {
 812               ti = nat_value_get_thread_index (&value0);
 813               if (ti != thread_index)
 814                 {
 815                   *required_thread_index = ti;
 816                   return 1;
 817                 }
 818               si = nat_value_get_session_index (&value0);
 819               s0 = pool_elt_at_index (nm->per_thread_data[ti].sessions, si);
 820               new_dst_addr0 = s0->in2out.addr.as_u32;
 821               vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 822               echo0->identifier = s0->in2out.port;
 823               sum0 = icmp0->checksum;
 824               sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 825                                      icmp_echo_header_t, identifier);
 826               icmp0->checksum = ip_csum_fold (sum0);
 827               goto change_addr;
 828             }
 829
 830           return 1;
 831         }
 832
 833       m0 = pool_elt_at_index (nm->static_mappings, value0.value);
 834
 835       new_dst_addr0 = m0->local_addr.as_u32;
 836       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 837         vnet_buffer (b0)->sw_if_index[VLIB_TX] = m0->fib_index;
 838     }
 839 change_addr:
 840   /* Destination is behind the same NAT, use internal address and port */
 841   if (new_dst_addr0)
 842     {
 843       old_dst_addr0 = ip0->dst_address.as_u32;
 844       ip0->dst_address.as_u32 = new_dst_addr0;
 845       sum0 = ip0->checksum;
 846       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0, ip4_header_t,
 847                              dst_address);
 848       ip0->checksum = ip_csum_fold (sum0);
 849     }
 850   return 0;
 851 }
 852
 853 static_always_inline u32
 854 nat44_ei_icmp_in2out (vlib_buffer_t *b0, ip4_header_t *ip0,
 855                       icmp46_header_t *icmp0, u32 sw_if_index0,
 856                       u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0,
 857                       u32 thread_index, nat44_ei_session_t **p_s0)
 858 {
 859   nat44_ei_main_t *nm = &nat44_ei_main;
 860   vlib_main_t *vm = vlib_get_main ();
 861   ip4_address_t addr;
 862   u16 port;
 863   u32 fib_index;
 864   nat_protocol_t proto;
 865   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 866   ip4_header_t *inner_ip0;
 867   void *l4_header = 0;
 868   icmp46_header_t *inner_icmp0;
 869   u8 dont_translate;
 870   u32 new_addr0, old_addr0;
 871   u16 old_id0, new_id0;
 872   u16 old_checksum0, new_checksum0;
 873   ip_csum_t sum0;
 874   u16 checksum0;
 875   u32 next0_tmp;
 876   u32 required_thread_index = thread_index;
 877
 878   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 879
 880   if (PREDICT_TRUE (nm->pat))
 881     {
 882       next0_tmp = nat44_ei_icmp_match_in2out_slow (
 883         node, thread_index, b0, ip0, &addr, &port, &fib_index, &proto, p_s0,
 884         &dont_translate);
 885     }
 886   else
 887     {
 888       next0_tmp = nat44_ei_icmp_match_in2out_fast (
 889         node, thread_index, b0, ip0, &addr, &port, &fib_index, &proto, p_s0,
 890         &dont_translate);
 891     }
 892
 893   if (next0_tmp != ~0)
 894     next0 = next0_tmp;
 895   if (next0 == NAT44_EI_IN2OUT_NEXT_DROP || dont_translate)
 896     goto out;
 897
 898   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 899     {
 900       sum0 =
 901         ip_incremental_checksum_buffer (vm, b0,
 902                                         (u8 *) icmp0 -
 903                                         (u8 *) vlib_buffer_get_current (b0),
 904                                         ntohs (ip0->length) -
 905                                         ip4_header_bytes (ip0), 0);
 906       checksum0 = ~ip_csum_fold (sum0);
 907       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 908         {
 909           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 910           goto out;
 911         }
 912     }
 913
 914   old_addr0 = ip0->src_address.as_u32;
 915   new_addr0 = ip0->src_address.as_u32 = addr.as_u32;
 916
 917   sum0 = ip0->checksum;
 918   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 919                          src_address /* changed member */ );
 920   ip0->checksum = ip_csum_fold (sum0);
 921
 922   if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 923     {
 924       if (icmp0->checksum == 0)
 925         icmp0->checksum = 0xffff;
 926
 927       if (!icmp_type_is_error_message (icmp0->type))
 928         {
 929           new_id0 = port;
 930           if (PREDICT_FALSE (new_id0 != echo0->identifier))
 931             {
 932               old_id0 = echo0->identifier;
 933               new_id0 = port;
 934               echo0->identifier = new_id0;
 935
 936               sum0 = icmp0->checksum;
 937               sum0 =
 938                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 939                                 identifier);
 940               icmp0->checksum = ip_csum_fold (sum0);
 941             }
 942         }
 943       else
 944         {
 945           inner_ip0 = (ip4_header_t *) (echo0 + 1);
 946           l4_header = ip4_next_header (inner_ip0);
 947
 948           if (!ip4_header_checksum_is_valid (inner_ip0))
 949             {
 950               next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 951               goto out;
 952             }
 953
 954           /* update inner destination IP address */
 955           old_addr0 = inner_ip0->dst_address.as_u32;
 956           inner_ip0->dst_address = addr;
 957           new_addr0 = inner_ip0->dst_address.as_u32;
 958           sum0 = icmp0->checksum;
 959           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 960                                  dst_address /* changed member */ );
 961           icmp0->checksum = ip_csum_fold (sum0);
 962
 963           /* update inner IP header checksum */
 964           old_checksum0 = inner_ip0->checksum;
 965           sum0 = inner_ip0->checksum;
 966           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 967                                  dst_address /* changed member */ );
 968           inner_ip0->checksum = ip_csum_fold (sum0);
 969           new_checksum0 = inner_ip0->checksum;
 970           sum0 = icmp0->checksum;
 971           sum0 =
 972             ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 973                             checksum);
 974           icmp0->checksum = ip_csum_fold (sum0);
 975
 976           switch (proto)
 977             {
 978             case NAT_PROTOCOL_ICMP:
 979               inner_icmp0 = (icmp46_header_t *) l4_header;
 980               inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 981
 982               old_id0 = inner_echo0->identifier;
 983               new_id0 = port;
 984               inner_echo0->identifier = new_id0;
 985
 986               sum0 = icmp0->checksum;
 987               sum0 =
 988                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 989                                 identifier);
 990               icmp0->checksum = ip_csum_fold (sum0);
 991               break;
 992             case NAT_PROTOCOL_UDP:
 993             case NAT_PROTOCOL_TCP:
 994               old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 995               new_id0 = port;
 996               ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 997
 998               sum0 = icmp0->checksum;
 999               sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
1000                                      dst_port);
1001               icmp0->checksum = ip_csum_fold (sum0);
1002               break;
1003             default:
1004               ASSERT (0);
1005             }
1006         }
1007     }
1008
1009   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
1010     {
1011       if (0 != nat44_ei_icmp_hairpinning (nm, b0, thread_index, ip0, icmp0,
1012                                           &required_thread_index))
1013         vnet_buffer (b0)->sw_if_index[VLIB_TX] = fib_index;
1014       if (thread_index != required_thread_index)
1015         {
1016           vnet_buffer (b0)->snat.required_thread_index = required_thread_index;
1017           next0 = NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF;
1018         }
1019     }
1020
1021 out:
1022   return next0;
1023 }
1024
1025 static_always_inline u32
1026 nat44_ei_icmp_in2out_slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0,
1027                                 ip4_header_t *ip0, icmp46_header_t *icmp0,
1028                                 u32 sw_if_index0, u32 rx_fib_index0,
1029                                 vlib_node_runtime_t *node, u32 next0, f64 now,
1030                                 u32 thread_index, nat44_ei_session_t **p_s0)
1031 {
1032   vlib_main_t *vm = vlib_get_main ();
1033
1034   next0 = nat44_ei_icmp_in2out (b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1035                                 node, next0, thread_index, p_s0);
1036   nat44_ei_session_t *s0 = *p_s0;
1037   if (PREDICT_TRUE (next0 != NAT44_EI_IN2OUT_NEXT_DROP && s0))
1038     {
1039       /* Accounting */
1040       nat44_ei_session_update_counters (
1041         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1042       /* Per-user LRU list maintenance */
1043       nat44_ei_session_update_lru (nm, s0, thread_index);
1044     }
1045   return next0;
1046 }
1047
1048 static_always_inline void
1049 nat44_ei_hairpinning_sm_unknown_proto (nat44_ei_main_t *nm, vlib_buffer_t *b,
1050                                        ip4_header_t *ip)
1051 {
1052   clib_bihash_kv_8_8_t kv, value;
1053   nat44_ei_static_mapping_t *m;
1054   u32 old_addr, new_addr;
1055   ip_csum_t sum;
1056
1057   init_nat_k (&kv, ip->dst_address, 0, 0, 0);
1058   if (clib_bihash_search_8_8 (&nm->static_mapping_by_external, &kv, &value))
1059     return;
1060
1061   m = pool_elt_at_index (nm->static_mappings, value.value);
1062
1063   old_addr = ip->dst_address.as_u32;
1064   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1065   sum = ip->checksum;
1066   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1067   ip->checksum = ip_csum_fold (sum);
1068
1069   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
1070     vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
1071 }
1072
1073 static int
1074 nat_in2out_sm_unknown_proto (nat44_ei_main_t *nm, vlib_buffer_t *b,
1075                              ip4_header_t *ip, u32 rx_fib_index)
1076 {
1077   clib_bihash_kv_8_8_t kv, value;
1078   nat44_ei_static_mapping_t *m;
1079   u32 old_addr, new_addr;
1080   ip_csum_t sum;
1081
1082   init_nat_k (&kv, ip->src_address, 0, rx_fib_index, 0);
1083   if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
1084     return 1;
1085
1086   m = pool_elt_at_index (nm->static_mappings, value.value);
1087
1088   old_addr = ip->src_address.as_u32;
1089   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1090   sum = ip->checksum;
1091   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1092   ip->checksum = ip_csum_fold (sum);
1093
1094
1095   /* Hairpinning */
1096   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
1097     {
1098       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
1099       nat44_ei_hairpinning_sm_unknown_proto (nm, b, ip);
1100     }
1101
1102   return 0;
1103 }
1104
1105 static_always_inline int
1106 nat44_ei_hairpinning (vlib_main_t *vm, vlib_node_runtime_t *node,
1107                       nat44_ei_main_t *nm, u32 thread_index, vlib_buffer_t *b0,
1108                       ip4_header_t *ip0, udp_header_t *udp0,
1109                       tcp_header_t *tcp0, u32 proto0, int do_trace,
1110                       u32 *required_thread_index)
1111 {
1112   nat44_ei_session_t *s0 = NULL;
1113   clib_bihash_kv_8_8_t kv0, value0;
1114   ip_csum_t sum0;
1115   u32 new_dst_addr0 = 0, old_dst_addr0, si = ~0;
1116   u16 new_dst_port0 = ~0, old_dst_port0;
1117   int rv;
1118   ip4_address_t sm0_addr;
1119   u16 sm0_port;
1120   u32 sm0_fib_index;
1121   u32 old_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1122
1123   /* Check if destination is static mappings */
1124   if (!nat44_ei_static_mapping_match (
1125         ip0->dst_address, udp0->dst_port, nm->outside_fib_index, proto0,
1126         &sm0_addr, &sm0_port, &sm0_fib_index, 1 /* by external */, 0, 0))
1127     {
1128       new_dst_addr0 = sm0_addr.as_u32;
1129       new_dst_port0 = sm0_port;
1130       vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0_fib_index;
1131     }
1132   /* or active session */
1133   else
1134     {
1135       init_nat_k (&kv0, ip0->dst_address, udp0->dst_port,
1136                   nm->outside_fib_index, proto0);
1137       rv = clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0);
1138       if (rv)
1139         {
1140           rv = 0;
1141           goto trace;
1142         }
1143
1144       if (thread_index != nat_value_get_thread_index (&value0))
1145         {
1146           *required_thread_index = nat_value_get_thread_index (&value0);
1147           return 0;
1148         }
1149
1150       si = nat_value_get_session_index (&value0);
1151       s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions, si);
1152       new_dst_addr0 = s0->in2out.addr.as_u32;
1153       new_dst_port0 = s0->in2out.port;
1154       vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1155     }
1156
1157   /* Check if anything has changed and if not, then return 0. This
1158      helps avoid infinite loop, repeating the three nodes
1159      nat44-hairpinning-->ip4-lookup-->ip4-local, in case nothing has
1160      changed. */
1161   old_dst_addr0 = ip0->dst_address.as_u32;
1162   old_dst_port0 = tcp0->dst;
1163   if (new_dst_addr0 == old_dst_addr0 && new_dst_port0 == old_dst_port0 &&
1164       vnet_buffer (b0)->sw_if_index[VLIB_TX] == old_sw_if_index)
1165     return 0;
1166
1167   /* Destination is behind the same NAT, use internal address and port */
1168   if (new_dst_addr0)
1169     {
1170       old_dst_addr0 = ip0->dst_address.as_u32;
1171       ip0->dst_address.as_u32 = new_dst_addr0;
1172       sum0 = ip0->checksum;
1173       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0, ip4_header_t,
1174                              dst_address);
1175       ip0->checksum = ip_csum_fold (sum0);
1176
1177       old_dst_port0 = tcp0->dst;
1178       if (PREDICT_TRUE (new_dst_port0 != old_dst_port0))
1179         {
1180           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1181             {
1182               tcp0->dst = new_dst_port0;
1183               sum0 = tcp0->checksum;
1184               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1185                                      ip4_header_t, dst_address);
1186               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
1187                                      ip4_header_t /* cheat */, length);
1188               tcp0->checksum = ip_csum_fold (sum0);
1189             }
1190           else
1191             {
1192               udp0->dst_port = new_dst_port0;
1193               udp0->checksum = 0;
1194             }
1195         }
1196       else
1197         {
1198           if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1199             {
1200               sum0 = tcp0->checksum;
1201               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1202                                      ip4_header_t, dst_address);
1203               tcp0->checksum = ip_csum_fold (sum0);
1204             }
1205         }
1206       rv = 1;
1207       goto trace;
1208     }
1209   rv = 0;
1210 trace:
1211   if (do_trace && PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
1212                                  (b0->flags & VLIB_BUFFER_IS_TRACED)))
1213     {
1214       nat44_ei_hairpin_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
1215       t->addr.as_u32 = new_dst_addr0;
1216       t->port = new_dst_port0;
1217       t->fib_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1218       if (s0)
1219         {
1220           t->session_index = si;
1221         }
1222       else
1223         {
1224           t->session_index = ~0;
1225         }
1226     }
1227   return rv;
1228 }
1229
1230 static_always_inline uword
1231 nat44_ei_hairpinning_handoff_fn_inline (vlib_main_t *vm,
1232                                         vlib_node_runtime_t *node,
1233                                         vlib_frame_t *frame, u32 fq_index)
1234 {
1235   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1236   u32 n_enq, n_left_from, *from;
1237   u16 thread_indices[VLIB_FRAME_SIZE], *ti;
1238
1239   from = vlib_frame_vector_args (frame);
1240   n_left_from = frame->n_vectors;
1241   vlib_get_buffers (vm, from, bufs, n_left_from);
1242
1243   b = bufs;
1244   ti = thread_indices;
1245
1246   while (n_left_from > 0)
1247     {
1248       ti[0] = vnet_buffer (b[0])->snat.required_thread_index;
1249
1250       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
1251                          (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1252         {
1253           nat44_ei_hairpinning_handoff_trace_t *t =
1254             vlib_add_trace (vm, node, b[0], sizeof (*t));
1255           t->next_worker_index = ti[0];
1256         }
1257
1258       n_left_from -= 1;
1259       ti += 1;
1260       b += 1;
1261     }
1262   n_enq = vlib_buffer_enqueue_to_thread (vm, node, fq_index, from,
1263                                          thread_indices, frame->n_vectors, 1);
1264
1265   if (n_enq < frame->n_vectors)
1266     vlib_node_increment_counter (
1267       vm, node->node_index, NAT44_EI_HAIRPINNING_HANDOFF_ERROR_CONGESTION_DROP,
1268       frame->n_vectors - n_enq);
1269   return frame->n_vectors;
1270 }
1271
1272 static_always_inline uword
1273 nat44_ei_in2out_node_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
1274                                 vlib_frame_t *frame, int is_slow_path,
1275                                 int is_output_feature)
1276 {
1277   u32 n_left_from, *from;
1278   nat44_ei_main_t *nm = &nat44_ei_main;
1279   f64 now = vlib_time_now (vm);
1280   u32 thread_index = vm->thread_index;
1281
1282   from = vlib_frame_vector_args (frame);
1283   n_left_from = frame->n_vectors;
1284
1285   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1286   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1287   vlib_get_buffers (vm, from, b, n_left_from);
1288
1289   while (n_left_from >= 2)
1290     {
1291       vlib_buffer_t *b0, *b1;
1292       u32 next0, next1;
1293       u32 rx_sw_if_index0, rx_sw_if_index1;
1294       u32 tx_sw_if_index0, tx_sw_if_index1;
1295       u32 cntr_sw_if_index0, cntr_sw_if_index1;
1296       ip4_header_t *ip0, *ip1;
1297       ip_csum_t sum0, sum1;
1298       u32 new_addr0, old_addr0, new_addr1, old_addr1;
1299       u16 old_port0, new_port0, old_port1, new_port1;
1300       udp_header_t *udp0, *udp1;
1301       tcp_header_t *tcp0, *tcp1;
1302       icmp46_header_t *icmp0, *icmp1;
1303       u32 rx_fib_index0, rx_fib_index1;
1304       u32 proto0, proto1;
1305       nat44_ei_session_t *s0 = 0, *s1 = 0;
1306       clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1307       u32 iph_offset0 = 0, iph_offset1 = 0;
1308
1309       b0 = *b;
1310       b++;
1311       b1 = *b;
1312       b++;
1313
1314       /* Prefetch next iteration. */
1315       if (PREDICT_TRUE (n_left_from >= 4))
1316         {
1317           vlib_buffer_t *p2, *p3;
1318
1319           p2 = *b;
1320           p3 = *(b + 1);
1321
1322           vlib_prefetch_buffer_header (p2, LOAD);
1323           vlib_prefetch_buffer_header (p3, LOAD);
1324
1325           clib_prefetch_load (p2->data);
1326           clib_prefetch_load (p3->data);
1327         }
1328
1329       if (is_output_feature)
1330         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1331
1332       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1333                               iph_offset0);
1334
1335       udp0 = ip4_next_header (ip0);
1336       tcp0 = (tcp_header_t *) udp0;
1337       icmp0 = (icmp46_header_t *) udp0;
1338
1339       rx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1340       tx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1341       cntr_sw_if_index0 =
1342         is_output_feature ? tx_sw_if_index0 : rx_sw_if_index0;
1343       rx_fib_index0 =
1344         vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index0);
1345
1346       next0 = next1 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
1347
1348       if (PREDICT_FALSE (ip0->ttl == 1))
1349         {
1350           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1351           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1352                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1353                                        0);
1354           next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1355           goto trace00;
1356         }
1357
1358       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1359
1360       /* Next configured feature, probably ip4-lookup */
1361       if (is_slow_path)
1362         {
1363           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1364             {
1365               if (nat_in2out_sm_unknown_proto (nm, b0, ip0, rx_fib_index0))
1366                 {
1367                   next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1368                   b0->error =
1369                     node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1370                 }
1371               vlib_increment_simple_counter (
1372                 is_slow_path ? &nm->counters.slowpath.in2out.other :
1373                                &nm->counters.fastpath.in2out.other,
1374                 thread_index, cntr_sw_if_index0, 1);
1375               goto trace00;
1376             }
1377
1378           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1379             {
1380               next0 = nat44_ei_icmp_in2out_slow_path (
1381                 nm, b0, ip0, icmp0, rx_sw_if_index0, rx_fib_index0, node,
1382                 next0, now, thread_index, &s0);
1383               vlib_increment_simple_counter (
1384                 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1385                                &nm->counters.fastpath.in2out.icmp,
1386                 thread_index, cntr_sw_if_index0, 1);
1387               goto trace00;
1388             }
1389         }
1390       else
1391         {
1392           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1393             {
1394               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1395               goto trace00;
1396             }
1397
1398           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1399             {
1400               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1401               goto trace00;
1402             }
1403         }
1404
1405       init_nat_k (&kv0, ip0->src_address,
1406                   vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1407                   proto0);
1408       if (PREDICT_FALSE (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0) !=
1409                          0))
1410         {
1411           if (is_slow_path)
1412             {
1413               if (is_output_feature)
1414                 {
1415                   if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1416                         nm, ip0, proto0,
1417                         vnet_buffer (b0)->ip.reass.l4_src_port,
1418                         vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1419                         rx_sw_if_index0)))
1420                     goto trace00;
1421
1422                   /*
1423                    * Send DHCP packets to the ipv4 stack, or we won't
1424                    * be able to use dhcp client on the outside interface
1425                    */
1426                   if (PREDICT_FALSE
1427                       (proto0 == NAT_PROTOCOL_UDP
1428                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1429                            clib_host_to_net_u16
1430                            (UDP_DST_PORT_dhcp_to_server))
1431                        && ip0->dst_address.as_u32 == 0xffffffff))
1432                     goto trace00;
1433                 }
1434               else
1435                 {
1436                   if (PREDICT_FALSE (nat44_ei_not_translate (
1437                         nm, node, rx_sw_if_index0, ip0, proto0, rx_fib_index0,
1438                         thread_index)))
1439                     goto trace00;
1440                 }
1441
1442               next0 = slow_path (nm, b0, ip0, ip0->src_address,
1443                                  vnet_buffer (b0)->ip.reass.l4_src_port,
1444                                  rx_fib_index0, proto0, &s0, node, next0,
1445                                  thread_index, now);
1446               if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
1447                 goto trace00;
1448
1449               if (PREDICT_FALSE (!s0))
1450                 goto trace00;
1451             }
1452           else
1453             {
1454               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1455               goto trace00;
1456             }
1457         }
1458       else
1459         s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1460                                 nat_value_get_session_index (&value0));
1461
1462       b0->flags |= VNET_BUFFER_F_IS_NATED;
1463
1464       old_addr0 = ip0->src_address.as_u32;
1465       ip0->src_address = s0->out2in.addr;
1466       new_addr0 = ip0->src_address.as_u32;
1467       if (!is_output_feature)
1468         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1469
1470       sum0 = ip0->checksum;
1471       sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1472                              ip4_header_t, src_address /* changed member */ );
1473       ip0->checksum = ip_csum_fold (sum0);
1474
1475
1476       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1477         {
1478           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1479             {
1480               old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1481               new_port0 = udp0->src_port = s0->out2in.port;
1482               sum0 = tcp0->checksum;
1483               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1484                                      ip4_header_t,
1485                                      dst_address /* changed member */ );
1486               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1487                                      ip4_header_t /* cheat */ ,
1488                                      length /* changed member */ );
1489               mss_clamping (nm->mss_clamping, tcp0, &sum0);
1490               tcp0->checksum = ip_csum_fold (sum0);
1491             }
1492           vlib_increment_simple_counter (is_slow_path ?
1493                                            &nm->counters.slowpath.in2out.tcp :
1494                                            &nm->counters.fastpath.in2out.tcp,
1495                                          thread_index, cntr_sw_if_index0, 1);
1496         }
1497       else
1498         {
1499           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1500             {
1501               udp0->src_port = s0->out2in.port;
1502               if (PREDICT_FALSE (udp0->checksum))
1503                 {
1504                   old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1505                   new_port0 = udp0->src_port;
1506                   sum0 = udp0->checksum;
1507                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address  /* changed member */
1508                     );
1509                   sum0 =
1510                     ip_csum_update (sum0, old_port0, new_port0,
1511                                     ip4_header_t /* cheat */ ,
1512                                     length /* changed member */ );
1513                   udp0->checksum = ip_csum_fold (sum0);
1514                 }
1515             }
1516           vlib_increment_simple_counter (is_slow_path ?
1517                                            &nm->counters.slowpath.in2out.udp :
1518                                            &nm->counters.fastpath.in2out.udp,
1519                                          thread_index, cntr_sw_if_index0, 1);
1520         }
1521
1522       /* Accounting */
1523       nat44_ei_session_update_counters (
1524         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1525       /* Per-user LRU list maintenance */
1526       nat44_ei_session_update_lru (nm, s0, thread_index);
1527     trace00:
1528
1529       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1530                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1531         {
1532           nat44_ei_in2out_trace_t *t =
1533             vlib_add_trace (vm, node, b0, sizeof (*t));
1534           t->is_slow_path = is_slow_path;
1535           t->sw_if_index = rx_sw_if_index0;
1536           t->next_index = next0;
1537           t->session_index = ~0;
1538           if (s0)
1539             t->session_index = s0 - nm->per_thread_data[thread_index].sessions;
1540         }
1541
1542       if (next0 == NAT44_EI_IN2OUT_NEXT_DROP)
1543         {
1544           vlib_increment_simple_counter (
1545             is_slow_path ? &nm->counters.slowpath.in2out.drops :
1546                            &nm->counters.fastpath.in2out.drops,
1547             thread_index, cntr_sw_if_index0, 1);
1548         }
1549
1550       if (is_output_feature)
1551         iph_offset1 = vnet_buffer (b1)->ip.reass.save_rewrite_length;
1552
1553       ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1554                               iph_offset1);
1555
1556       udp1 = ip4_next_header (ip1);
1557       tcp1 = (tcp_header_t *) udp1;
1558       icmp1 = (icmp46_header_t *) udp1;
1559
1560       rx_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1561       tx_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_TX];
1562       cntr_sw_if_index1 =
1563         is_output_feature ? tx_sw_if_index1 : rx_sw_if_index1;
1564       rx_fib_index1 =
1565         vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index1);
1566
1567       if (PREDICT_FALSE (ip1->ttl == 1))
1568         {
1569           vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1570           icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1571                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1572                                        0);
1573           next1 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1574           goto trace01;
1575         }
1576
1577       proto1 = ip_proto_to_nat_proto (ip1->protocol);
1578
1579       /* Next configured feature, probably ip4-lookup */
1580       if (is_slow_path)
1581         {
1582           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1583             {
1584               if (nat_in2out_sm_unknown_proto (nm, b1, ip1, rx_fib_index1))
1585                 {
1586                   next1 = NAT44_EI_IN2OUT_NEXT_DROP;
1587                   b1->error =
1588                     node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1589                 }
1590               vlib_increment_simple_counter (
1591                 is_slow_path ? &nm->counters.slowpath.in2out.other :
1592                                &nm->counters.fastpath.in2out.other,
1593                 thread_index, cntr_sw_if_index1, 1);
1594               goto trace01;
1595             }
1596
1597           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1598             {
1599               next1 = nat44_ei_icmp_in2out_slow_path (
1600                 nm, b1, ip1, icmp1, rx_sw_if_index1, rx_fib_index1, node,
1601                 next1, now, thread_index, &s1);
1602               vlib_increment_simple_counter (
1603                 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1604                                &nm->counters.fastpath.in2out.icmp,
1605                 thread_index, cntr_sw_if_index1, 1);
1606               goto trace01;
1607             }
1608         }
1609       else
1610         {
1611           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1612             {
1613               next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1614               goto trace01;
1615             }
1616
1617           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1618             {
1619               next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1620               goto trace01;
1621             }
1622         }
1623
1624       init_nat_k (&kv1, ip1->src_address,
1625                   vnet_buffer (b1)->ip.reass.l4_src_port, rx_fib_index1,
1626                   proto1);
1627       if (PREDICT_FALSE (clib_bihash_search_8_8 (&nm->in2out, &kv1, &value1) !=
1628                          0))
1629         {
1630           if (is_slow_path)
1631             {
1632               if (is_output_feature)
1633                 {
1634                   if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1635                         nm, ip1, proto1,
1636                         vnet_buffer (b1)->ip.reass.l4_src_port,
1637                         vnet_buffer (b1)->ip.reass.l4_dst_port, thread_index,
1638                         rx_sw_if_index1)))
1639                     goto trace01;
1640
1641                   /*
1642                    * Send DHCP packets to the ipv4 stack, or we won't
1643                    * be able to use dhcp client on the outside interface
1644                    */
1645                   if (PREDICT_FALSE
1646                       (proto1 == NAT_PROTOCOL_UDP
1647                        && (vnet_buffer (b1)->ip.reass.l4_dst_port ==
1648                            clib_host_to_net_u16
1649                            (UDP_DST_PORT_dhcp_to_server))
1650                        && ip1->dst_address.as_u32 == 0xffffffff))
1651                     goto trace01;
1652                 }
1653               else
1654                 {
1655                   if (PREDICT_FALSE (nat44_ei_not_translate (
1656                         nm, node, rx_sw_if_index1, ip1, proto1, rx_fib_index1,
1657                         thread_index)))
1658                     goto trace01;
1659                 }
1660
1661               next1 = slow_path (nm, b1, ip1, ip1->src_address,
1662                                  vnet_buffer (b1)->ip.reass.l4_src_port,
1663                                  rx_fib_index1, proto1, &s1, node, next1,
1664                                  thread_index, now);
1665               if (PREDICT_FALSE (next1 == NAT44_EI_IN2OUT_NEXT_DROP))
1666                 goto trace01;
1667
1668               if (PREDICT_FALSE (!s1))
1669                 goto trace01;
1670             }
1671           else
1672             {
1673               next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1674               goto trace01;
1675             }
1676         }
1677       else
1678         s1 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1679                                 nat_value_get_session_index (&value1));
1680
1681       b1->flags |= VNET_BUFFER_F_IS_NATED;
1682
1683       old_addr1 = ip1->src_address.as_u32;
1684       ip1->src_address = s1->out2in.addr;
1685       new_addr1 = ip1->src_address.as_u32;
1686       if (!is_output_feature)
1687         vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1688
1689       sum1 = ip1->checksum;
1690       sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1691                              ip4_header_t, src_address /* changed member */ );
1692       ip1->checksum = ip_csum_fold (sum1);
1693
1694       if (PREDICT_TRUE (proto1 == NAT_PROTOCOL_TCP))
1695         {
1696           if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1697             {
1698               old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1699               new_port1 = udp1->src_port = s1->out2in.port;
1700               sum1 = tcp1->checksum;
1701               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1702                                      ip4_header_t,
1703                                      dst_address /* changed member */ );
1704               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1705                                      ip4_header_t /* cheat */ ,
1706                                      length /* changed member */ );
1707               mss_clamping (nm->mss_clamping, tcp1, &sum1);
1708               tcp1->checksum = ip_csum_fold (sum1);
1709             }
1710           vlib_increment_simple_counter (is_slow_path ?
1711                                            &nm->counters.slowpath.in2out.tcp :
1712                                            &nm->counters.fastpath.in2out.tcp,
1713                                          thread_index, cntr_sw_if_index1, 1);
1714         }
1715       else
1716         {
1717           if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1718             {
1719               udp1->src_port = s1->out2in.port;
1720               if (PREDICT_FALSE (udp1->checksum))
1721                 {
1722                   old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1723                   new_port1 = udp1->src_port;
1724                   sum1 = udp1->checksum;
1725                   sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t, dst_address  /* changed member */
1726                     );
1727                   sum1 =
1728                     ip_csum_update (sum1, old_port1, new_port1,
1729                                     ip4_header_t /* cheat */ ,
1730                                     length /* changed member */ );
1731                   udp1->checksum = ip_csum_fold (sum1);
1732                 }
1733             }
1734           vlib_increment_simple_counter (is_slow_path ?
1735                                            &nm->counters.slowpath.in2out.udp :
1736                                            &nm->counters.fastpath.in2out.udp,
1737                                          thread_index, cntr_sw_if_index1, 1);
1738         }
1739
1740       /* Accounting */
1741       nat44_ei_session_update_counters (
1742         s1, now, vlib_buffer_length_in_chain (vm, b1), thread_index);
1743       /* Per-user LRU list maintenance */
1744       nat44_ei_session_update_lru (nm, s1, thread_index);
1745     trace01:
1746
1747       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1748                          && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1749         {
1750           nat44_ei_in2out_trace_t *t =
1751             vlib_add_trace (vm, node, b1, sizeof (*t));
1752           t->sw_if_index = rx_sw_if_index1;
1753           t->next_index = next1;
1754           t->session_index = ~0;
1755           if (s1)
1756             t->session_index = s1 - nm->per_thread_data[thread_index].sessions;
1757         }
1758
1759       if (next1 == NAT44_EI_IN2OUT_NEXT_DROP)
1760         {
1761           vlib_increment_simple_counter (
1762             is_slow_path ? &nm->counters.slowpath.in2out.drops :
1763                            &nm->counters.fastpath.in2out.drops,
1764             thread_index, cntr_sw_if_index1, 1);
1765         }
1766
1767       n_left_from -= 2;
1768       next[0] = next0;
1769       next[1] = next1;
1770       next += 2;
1771     }
1772
1773   while (n_left_from > 0)
1774     {
1775       vlib_buffer_t *b0;
1776       u32 next0;
1777       u32 rx_sw_if_index0;
1778       u32 tx_sw_if_index0;
1779       u32 cntr_sw_if_index0;
1780       ip4_header_t *ip0;
1781       ip_csum_t sum0;
1782       u32 new_addr0, old_addr0;
1783       u16 old_port0, new_port0;
1784       udp_header_t *udp0;
1785       tcp_header_t *tcp0;
1786       icmp46_header_t *icmp0;
1787       u32 rx_fib_index0;
1788       u32 proto0;
1789       nat44_ei_session_t *s0 = 0;
1790       clib_bihash_kv_8_8_t kv0, value0;
1791       u32 iph_offset0 = 0;
1792
1793       b0 = *b;
1794       b++;
1795       next0 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
1796
1797       if (is_output_feature)
1798         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1799
1800       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1801                               iph_offset0);
1802
1803       udp0 = ip4_next_header (ip0);
1804       tcp0 = (tcp_header_t *) udp0;
1805       icmp0 = (icmp46_header_t *) udp0;
1806
1807       rx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1808       tx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1809       cntr_sw_if_index0 =
1810         is_output_feature ? tx_sw_if_index0 : rx_sw_if_index0;
1811       rx_fib_index0 =
1812         vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index0);
1813
1814       if (PREDICT_FALSE (ip0->ttl == 1))
1815         {
1816           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1817           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1818                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1819                                        0);
1820           next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1821           goto trace0;
1822         }
1823
1824       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1825
1826       /* Next configured feature, probably ip4-lookup */
1827       if (is_slow_path)
1828         {
1829           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1830             {
1831               if (nat_in2out_sm_unknown_proto (nm, b0, ip0, rx_fib_index0))
1832                 {
1833                   next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1834                   b0->error =
1835                     node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1836                 }
1837               vlib_increment_simple_counter (
1838                 is_slow_path ? &nm->counters.slowpath.in2out.other :
1839                                &nm->counters.fastpath.in2out.other,
1840                 thread_index, cntr_sw_if_index0, 1);
1841               goto trace0;
1842             }
1843
1844           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1845             {
1846               next0 = nat44_ei_icmp_in2out_slow_path (
1847                 nm, b0, ip0, icmp0, rx_sw_if_index0, rx_fib_index0, node,
1848                 next0, now, thread_index, &s0);
1849               vlib_increment_simple_counter (
1850                 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1851                                &nm->counters.fastpath.in2out.icmp,
1852                 thread_index, cntr_sw_if_index0, 1);
1853               goto trace0;
1854             }
1855         }
1856       else
1857         {
1858           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1859             {
1860               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1861               goto trace0;
1862             }
1863
1864           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1865             {
1866               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1867               goto trace0;
1868             }
1869         }
1870
1871       init_nat_k (&kv0, ip0->src_address,
1872                   vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1873                   proto0);
1874
1875       if (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
1876         {
1877           if (is_slow_path)
1878             {
1879               if (is_output_feature)
1880                 {
1881                   if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1882                         nm, ip0, proto0,
1883                         vnet_buffer (b0)->ip.reass.l4_src_port,
1884                         vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1885                         rx_sw_if_index0)))
1886                     goto trace0;
1887
1888                   /*
1889                    * Send DHCP packets to the ipv4 stack, or we won't
1890                    * be able to use dhcp client on the outside interface
1891                    */
1892                   if (PREDICT_FALSE
1893                       (proto0 == NAT_PROTOCOL_UDP
1894                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1895                            clib_host_to_net_u16
1896                            (UDP_DST_PORT_dhcp_to_server))
1897                        && ip0->dst_address.as_u32 == 0xffffffff))
1898                     goto trace0;
1899                 }
1900               else
1901                 {
1902                   if (PREDICT_FALSE (nat44_ei_not_translate (
1903                         nm, node, rx_sw_if_index0, ip0, proto0, rx_fib_index0,
1904                         thread_index)))
1905                     goto trace0;
1906                 }
1907
1908               next0 = slow_path (nm, b0, ip0, ip0->src_address,
1909                                  vnet_buffer (b0)->ip.reass.l4_src_port,
1910                                  rx_fib_index0, proto0, &s0, node, next0,
1911                                  thread_index, now);
1912
1913               if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
1914                 goto trace0;
1915
1916               if (PREDICT_FALSE (!s0))
1917                 goto trace0;
1918             }
1919           else
1920             {
1921               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1922               goto trace0;
1923             }
1924         }
1925       else
1926         s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1927                                 nat_value_get_session_index (&value0));
1928
1929       b0->flags |= VNET_BUFFER_F_IS_NATED;
1930
1931       old_addr0 = ip0->src_address.as_u32;
1932       ip0->src_address = s0->out2in.addr;
1933       new_addr0 = ip0->src_address.as_u32;
1934       if (!is_output_feature)
1935         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1936
1937       sum0 = ip0->checksum;
1938       sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1939                              ip4_header_t, src_address /* changed member */ );
1940       ip0->checksum = ip_csum_fold (sum0);
1941
1942       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1943         {
1944           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1945             {
1946               old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1947               new_port0 = udp0->src_port = s0->out2in.port;
1948               sum0 = tcp0->checksum;
1949               sum0 =
1950                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1951                                 dst_address /* changed member */ );
1952               sum0 =
1953                 ip_csum_update (sum0, old_port0, new_port0,
1954                                 ip4_header_t /* cheat */ ,
1955                                 length /* changed member */ );
1956               mss_clamping (nm->mss_clamping, tcp0, &sum0);
1957               tcp0->checksum = ip_csum_fold (sum0);
1958             }
1959           vlib_increment_simple_counter (is_slow_path ?
1960                                            &nm->counters.slowpath.in2out.tcp :
1961                                            &nm->counters.fastpath.in2out.tcp,
1962                                          thread_index, cntr_sw_if_index0, 1);
1963         }
1964       else
1965         {
1966           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1967             {
1968               udp0->src_port = s0->out2in.port;
1969               if (PREDICT_FALSE (udp0->checksum))
1970                 {
1971                   old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1972                   new_port0 = udp0->src_port;
1973                   sum0 = udp0->checksum;
1974                   sum0 =
1975                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1976                                     dst_address /* changed member */ );
1977                   sum0 =
1978                     ip_csum_update (sum0, old_port0, new_port0,
1979                                     ip4_header_t /* cheat */ ,
1980                                     length /* changed member */ );
1981                   udp0->checksum = ip_csum_fold (sum0);
1982                 }
1983             }
1984           vlib_increment_simple_counter (is_slow_path ?
1985                                            &nm->counters.slowpath.in2out.udp :
1986                                            &nm->counters.fastpath.in2out.udp,
1987                                          thread_index, cntr_sw_if_index0, 1);
1988         }
1989
1990       /* Accounting */
1991       nat44_ei_session_update_counters (
1992         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1993       /* Per-user LRU list maintenance */
1994       nat44_ei_session_update_lru (nm, s0, thread_index);
1995
1996     trace0:
1997       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1998                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1999         {
2000           nat44_ei_in2out_trace_t *t =
2001             vlib_add_trace (vm, node, b0, sizeof (*t));
2002           t->is_slow_path = is_slow_path;
2003           t->sw_if_index = rx_sw_if_index0;
2004           t->next_index = next0;
2005           t->session_index = ~0;
2006           if (s0)
2007             t->session_index = s0 - nm->per_thread_data[thread_index].sessions;
2008         }
2009
2010       if (next0 == NAT44_EI_IN2OUT_NEXT_DROP)
2011         {
2012           vlib_increment_simple_counter (
2013             is_slow_path ? &nm->counters.slowpath.in2out.drops :
2014                            &nm->counters.fastpath.in2out.drops,
2015             thread_index, cntr_sw_if_index0, 1);
2016         }
2017
2018       n_left_from--;
2019       next[0] = next0;
2020       next++;
2021     }
2022
2023   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
2024                                frame->n_vectors);
2025   return frame->n_vectors;
2026 }
2027
2028 static_always_inline uword
2029 nat44_ei_in2out_hairpinning_finish_inline (vlib_main_t *vm,
2030                                            vlib_node_runtime_t *node,
2031                                            vlib_frame_t *frame)
2032 {
2033   u32 n_left_from, *from, *to_next;
2034   u32 thread_index = vm->thread_index;
2035   nat44_ei_in2out_next_t next_index;
2036   nat44_ei_main_t *nm = &nat44_ei_main;
2037   int is_hairpinning = 0;
2038
2039   from = vlib_frame_vector_args (frame);
2040   n_left_from = frame->n_vectors;
2041   next_index = node->cached_next_index;
2042
2043   while (n_left_from > 0)
2044     {
2045       u32 n_left_to_next;
2046
2047       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2048
2049       while (n_left_from > 0 && n_left_to_next > 0)
2050         {
2051           u32 bi0;
2052           vlib_buffer_t *b0;
2053           u32 next0;
2054           u32 sw_if_index0;
2055           ip4_header_t *ip0;
2056           udp_header_t *udp0;
2057           tcp_header_t *tcp0;
2058           icmp46_header_t *icmp0;
2059           u32 proto0;
2060           u32 required_thread_index = thread_index;
2061
2062           bi0 = from[0];
2063           to_next[0] = bi0;
2064           from += 1;
2065           to_next += 1;
2066           n_left_from -= 1;
2067           n_left_to_next -= 1;
2068
2069           b0 = vlib_get_buffer (vm, bi0);
2070           next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP;
2071
2072           ip0 = vlib_buffer_get_current (b0);
2073           udp0 = ip4_next_header (ip0);
2074           tcp0 = (tcp_header_t *) udp0;
2075           icmp0 = (icmp46_header_t *) udp0;
2076
2077           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2078           proto0 = ip_proto_to_nat_proto (ip0->protocol);
2079
2080           switch (proto0)
2081             {
2082             case NAT_PROTOCOL_TCP:
2083               // fallthrough
2084             case NAT_PROTOCOL_UDP:
2085               is_hairpinning = nat44_ei_hairpinning (
2086                 vm, node, nm, thread_index, b0, ip0, udp0, tcp0, proto0,
2087                 0 /* do_trace */, &required_thread_index);
2088               break;
2089             case NAT_PROTOCOL_ICMP:
2090               is_hairpinning = (0 == nat44_ei_icmp_hairpinning (
2091                                        nm, b0, thread_index, ip0, icmp0,
2092                                        &required_thread_index));
2093               break;
2094             case NAT_PROTOCOL_OTHER:
2095               // this should never happen
2096               next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2097               break;
2098             }
2099
2100           if (thread_index != required_thread_index)
2101             {
2102               // but we already did a handoff ...
2103               next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2104             }
2105
2106           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
2107                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
2108             {
2109               nat44_ei_in2out_trace_t *t =
2110                 vlib_add_trace (vm, node, b0, sizeof (*t));
2111               t->sw_if_index = sw_if_index0;
2112               t->next_index = next0;
2113               t->is_hairpinning = is_hairpinning;
2114             }
2115
2116           if (next0 != NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP)
2117             {
2118               vlib_increment_simple_counter (
2119                 &nm->counters.fastpath.in2out.other, sw_if_index0,
2120                 vm->thread_index, 1);
2121             }
2122
2123           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2124                                            n_left_to_next, bi0, next0);
2125         }
2126
2127       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2128     }
2129
2130   return frame->n_vectors;
2131 }
2132
2133 VLIB_NODE_FN (nat44_ei_hairpinning_node)
2134 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2135 {
2136   u32 n_left_from, *from, *to_next;
2137   u32 thread_index = vm->thread_index;
2138   nat44_ei_hairpin_next_t next_index;
2139   nat44_ei_main_t *nm = &nat44_ei_main;
2140   vnet_feature_main_t *fm = &feature_main;
2141   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2142   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2143
2144   from = vlib_frame_vector_args (frame);
2145   n_left_from = frame->n_vectors;
2146   next_index = node->cached_next_index;
2147
2148   while (n_left_from > 0)
2149     {
2150       u32 n_left_to_next;
2151
2152       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2153
2154       while (n_left_from > 0 && n_left_to_next > 0)
2155         {
2156           u32 bi0;
2157           vlib_buffer_t *b0;
2158           u32 next0;
2159           ip4_header_t *ip0;
2160           u32 proto0;
2161           udp_header_t *udp0;
2162           tcp_header_t *tcp0;
2163           u32 sw_if_index0;
2164           u32 required_thread_index = thread_index;
2165
2166           bi0 = from[0];
2167           to_next[0] = bi0;
2168           from += 1;
2169           to_next += 1;
2170           n_left_from -= 1;
2171           n_left_to_next -= 1;
2172
2173           b0 = vlib_get_buffer (vm, bi0);
2174           ip0 = vlib_buffer_get_current (b0);
2175           udp0 = ip4_next_header (ip0);
2176           tcp0 = (tcp_header_t *) udp0;
2177           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2178
2179           proto0 = ip_proto_to_nat_proto (ip0->protocol);
2180           int next0_resolved = 0;
2181
2182           if (nat44_ei_hairpinning (vm, node, nm, thread_index, b0, ip0, udp0,
2183                                     tcp0, proto0, 1, &required_thread_index))
2184             {
2185               next0 = NAT44_EI_HAIRPIN_NEXT_LOOKUP;
2186               next0_resolved = 1;
2187             }
2188
2189           if (thread_index != required_thread_index)
2190             {
2191               vnet_buffer (b0)->snat.required_thread_index =
2192                 required_thread_index;
2193               next0 = NAT44_EI_HAIRPIN_NEXT_HANDOFF;
2194               next0_resolved = 1;
2195             }
2196
2197           if (!next0_resolved)
2198             vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2199                                   &next0, 0);
2200
2201           if (next0 != NAT44_EI_HAIRPIN_NEXT_DROP)
2202             {
2203               vlib_increment_simple_counter (
2204                 &nm->counters.hairpinning, vm->thread_index, sw_if_index0, 1);
2205             }
2206
2207           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2208                                            n_left_to_next, bi0, next0);
2209         }
2210
2211       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2212     }
2213
2214   return frame->n_vectors;
2215 }
2216
2217 VLIB_NODE_FN (nat44_ei_in2out_node)
2218 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2219 {
2220   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 0, 0);
2221 }
2222
2223 VLIB_NODE_FN (nat44_ei_in2out_output_node)
2224 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2225 {
2226   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 0, 1);
2227 }
2228
2229 VLIB_NODE_FN (nat44_ei_in2out_slowpath_node)
2230 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2231 {
2232   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 1, 0);
2233 }
2234
2235 VLIB_NODE_FN (nat44_ei_in2out_output_slowpath_node)
2236 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2237 {
2238   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 1, 1);
2239 }
2240
2241 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_handoff_ip4_lookup_node)
2242 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2243 {
2244   return nat44_ei_hairpinning_handoff_fn_inline (
2245     vm, node, frame,
2246     nat44_ei_main.in2out_hairpinning_finish_ip4_lookup_node_fq_index);
2247 }
2248
2249 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_handoff_interface_output_node)
2250 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2251 {
2252   return nat44_ei_hairpinning_handoff_fn_inline (
2253     vm, node, frame,
2254     nat44_ei_main.in2out_hairpinning_finish_interface_output_node_fq_index);
2255 }
2256
2257 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_finish_ip4_lookup_node)
2258 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2259 {
2260   return nat44_ei_in2out_hairpinning_finish_inline (vm, node, frame);
2261 }
2262
2263 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_finish_interface_output_node)
2264 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2265 {
2266   return nat44_ei_in2out_hairpinning_finish_inline (vm, node, frame);
2267 }
2268
2269 VLIB_NODE_FN (nat44_ei_hairpinning_handoff_node)
2270 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2271 {
2272   return nat44_ei_hairpinning_handoff_fn_inline (
2273     vm, node, frame, nat44_ei_main.hairpinning_fq_index);
2274 }
2275
2276 VLIB_REGISTER_NODE (nat44_ei_in2out_node) = {
2277   .name = "nat44-ei-in2out",
2278   .vector_size = sizeof (u32),
2279   .format_trace = format_nat44_ei_in2out_trace,
2280   .type = VLIB_NODE_TYPE_INTERNAL,
2281   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2282   .error_strings = nat44_ei_in2out_error_strings,
2283   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2284   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
2285   .next_nodes = {
2286     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
2287     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2288     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
2289     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2290     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2291   },
2292 };
2293
2294 VLIB_REGISTER_NODE (nat44_ei_in2out_output_node) = {
2295   .name = "nat44-ei-in2out-output",
2296   .vector_size = sizeof (u32),
2297   .format_trace = format_nat44_ei_in2out_trace,
2298   .type = VLIB_NODE_TYPE_INTERNAL,
2299   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2300   .error_strings = nat44_ei_in2out_error_strings,
2301   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2302   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
2303   .next_nodes = {
2304     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
2305     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "interface-output",
2306     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-output-slowpath",
2307     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2308     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-interface-output",
2309   },
2310 };
2311
2312 VLIB_REGISTER_NODE (nat44_ei_in2out_slowpath_node) = {
2313   .name = "nat44-ei-in2out-slowpath",
2314   .vector_size = sizeof (u32),
2315   .format_trace = format_nat44_ei_in2out_trace,
2316   .type = VLIB_NODE_TYPE_INTERNAL,
2317   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2318   .error_strings = nat44_ei_in2out_error_strings,
2319   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2320   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
2321   .next_nodes = {
2322     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
2323     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2324     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
2325     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2326     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2327   },
2328 };
2329
2330 VLIB_REGISTER_NODE (nat44_ei_in2out_output_slowpath_node) = {
2331   .name = "nat44-ei-in2out-output-slowpath",
2332   .vector_size = sizeof (u32),
2333   .format_trace = format_nat44_ei_in2out_trace,
2334   .type = VLIB_NODE_TYPE_INTERNAL,
2335   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2336   .error_strings = nat44_ei_in2out_error_strings,
2337   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2338   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
2339   .next_nodes = {
2340     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
2341     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "interface-output",
2342     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-output-slowpath",
2343     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2344     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-interface-output",
2345   },
2346 };
2347
2348 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_handoff_ip4_lookup_node) = {
2349   .name = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2350   .vector_size = sizeof (u32),
2351   .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2352   .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2353   .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2354   .n_next_nodes = 1,
2355   .next_nodes = {
2356     [0] = "error-drop",
2357   },
2358 };
2359
2360 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_handoff_interface_output_node) = {
2361   .name = "nat44-ei-in2out-hairpinning-handoff-interface-output",
2362   .vector_size = sizeof (u32),
2363   .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2364   .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2365   .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2366   .n_next_nodes = 1,
2367   .next_nodes = {
2368     [0] = "error-drop",
2369   },
2370 };
2371
2372 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_finish_ip4_lookup_node) = {
2373   .name = "nat44-ei-in2out-hairpinning-finish-ip4-lookup",
2374   .vector_size = sizeof (u32),
2375   .format_trace = format_nat44_ei_in2out_fast_trace,
2376   .type = VLIB_NODE_TYPE_INTERNAL,
2377   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2378   .error_strings = nat44_ei_in2out_error_strings,
2379   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2380   .n_next_nodes = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2381   .next_nodes = {
2382     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2383     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "ip4-lookup",
2384   },
2385 };
2386
2387 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_finish_interface_output_node) = {
2388   .name = "nat44-ei-in2out-hairpinning-finish-interface-output",
2389   .vector_size = sizeof (u32),
2390   .format_trace = format_nat44_ei_in2out_fast_trace,
2391   .type = VLIB_NODE_TYPE_INTERNAL,
2392   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2393   .error_strings = nat44_ei_in2out_error_strings,
2394   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2395   .n_next_nodes = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2396   .next_nodes = {
2397     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2398     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "interface-output",
2399   },
2400 };
2401
2402 VLIB_REGISTER_NODE (nat44_ei_hairpinning_handoff_node) = {
2403   .name = "nat44-ei-hairpinning-handoff",
2404   .vector_size = sizeof (u32),
2405   .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2406   .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2407   .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2408   .n_next_nodes = 1,
2409   .next_nodes = {
2410     [0] = "error-drop",
2411   },
2412 };
2413
2414 VLIB_REGISTER_NODE (nat44_ei_hairpinning_node) = {
2415   .name = "nat44-ei-hairpinning",
2416   .vector_size = sizeof (u32),
2417   .type = VLIB_NODE_TYPE_INTERNAL,
2418   .format_trace = format_nat44_ei_hairpin_trace,
2419   .n_next_nodes = NAT44_EI_HAIRPIN_N_NEXT,
2420   .next_nodes = {
2421     [NAT44_EI_HAIRPIN_NEXT_DROP] = "error-drop",
2422     [NAT44_EI_HAIRPIN_NEXT_LOOKUP] = "ip4-lookup",
2423     [NAT44_EI_HAIRPIN_NEXT_HANDOFF] = "nat44-ei-hairpinning-handoff",
2424   },
2425 };
2426
2427 /*
2428  * fd.io coding-style-patch-verification: ON
2429  *
2430  * Local Variables:
2431  * eval: (c-set-style "gnu")
2432  * End:
2433  */