src/plugins/nat/nat44-ei/nat44_ei_in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 EI inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21
  22 #include <vnet/vnet.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/udp/udp_local.h>
  26 #include <vnet/fib/ip4_fib.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/ipfix_logging.h>
  34 #include <nat/lib/nat_inlines.h>
  35 #include <nat/nat44-ei/nat44_ei_inlines.h>
  36 #include <nat/nat44-ei/nat44_ei.h>
  37 #include <nat/nat44-ei/nat44_ei_hairpinning.h>
  38
  39 typedef struct
  40 {
  41   u32 sw_if_index;
  42   u32 next_index;
  43   u32 session_index;
  44   u32 is_slow_path;
  45   u32 is_hairpinning;
  46 } nat44_ei_in2out_trace_t;
  47
  48 /* packet trace format function */
  49 static u8 *
  50 format_nat44_ei_in2out_trace (u8 *s, va_list *args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   nat44_ei_in2out_trace_t *t = va_arg (*args, nat44_ei_in2out_trace_t *);
  55   char *tag;
  56
  57   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  58
  59   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  60               t->sw_if_index, t->next_index, t->session_index);
  61   if (t->is_hairpinning)
  62     {
  63       s = format (s, ", with-hairpinning");
  64     }
  65
  66   return s;
  67 }
  68
  69 static u8 *
  70 format_nat44_ei_in2out_fast_trace (u8 *s, va_list *args)
  71 {
  72   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  73   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  74   nat44_ei_in2out_trace_t *t = va_arg (*args, nat44_ei_in2out_trace_t *);
  75
  76   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  77               t->sw_if_index, t->next_index);
  78
  79   return s;
  80 }
  81
  82 #define foreach_nat44_ei_in2out_error                                         \
  83   _ (UNSUPPORTED_PROTOCOL, "unsupported protocol")                            \
  84   _ (OUT_OF_PORTS, "out of ports")                                            \
  85   _ (BAD_OUTSIDE_FIB, "outside VRF ID not found")                             \
  86   _ (BAD_ICMP_TYPE, "unsupported ICMP type")                                  \
  87   _ (NO_TRANSLATION, "no translation")                                        \
  88   _ (MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")                      \
  89   _ (CANNOT_CREATE_USER, "cannot create NAT user")
  90
  91 typedef enum
  92 {
  93 #define _(sym, str) NAT44_EI_IN2OUT_ERROR_##sym,
  94   foreach_nat44_ei_in2out_error
  95 #undef _
  96     NAT44_EI_IN2OUT_N_ERROR,
  97 } nat44_ei_in2out_error_t;
  98
  99 static char *nat44_ei_in2out_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_nat44_ei_in2out_error
 102 #undef _
 103 };
 104
 105 typedef enum
 106 {
 107   NAT44_EI_IN2OUT_NEXT_LOOKUP,
 108   NAT44_EI_IN2OUT_NEXT_DROP,
 109   NAT44_EI_IN2OUT_NEXT_ICMP_ERROR,
 110   NAT44_EI_IN2OUT_NEXT_SLOW_PATH,
 111   NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF,
 112   NAT44_EI_IN2OUT_N_NEXT,
 113 } nat44_ei_in2out_next_t;
 114
 115 typedef enum
 116 {
 117   NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP,
 118   NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP,
 119   NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
 120 } nat44_ei_in2out_hairpinnig_finish_next_t;
 121
 122 static inline int
 123 nat44_ei_not_translate_fast (vlib_node_runtime_t *node, u32 sw_if_index0,
 124                              ip4_header_t *ip0, u32 proto0, u32 rx_fib_index0)
 125 {
 126   nat44_ei_main_t *nm = &nat44_ei_main;
 127
 128   if (nm->out2in_dpo)
 129     return 0;
 130
 131   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 132   nat44_ei_outside_fib_t *outside_fib;
 133   fib_prefix_t pfx = {
 134     .fp_proto = FIB_PROTOCOL_IP4,
 135     .fp_len = 32,
 136     .fp_addr = {
 137                 .ip4.as_u32 = ip0->dst_address.as_u32,
 138                 }
 139     ,
 140   };
 141
 142   /* Don't NAT packet aimed at the intfc address */
 143   if (PREDICT_FALSE (nat44_ei_is_interface_addr (
 144         nm->ip4_main, node, sw_if_index0, ip0->dst_address.as_u32)))
 145     return 1;
 146
 147   fei = fib_table_lookup (rx_fib_index0, &pfx);
 148   if (FIB_NODE_INDEX_INVALID != fei)
 149     {
 150       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 151       if (sw_if_index == ~0)
 152         {
 153           vec_foreach (outside_fib, nm->outside_fibs)
 154             {
 155               fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 156               if (FIB_NODE_INDEX_INVALID != fei)
 157                 {
 158                   sw_if_index = fib_entry_get_resolving_interface (fei);
 159                   if (sw_if_index != ~0)
 160                     break;
 161                 }
 162             }
 163         }
 164       if (sw_if_index == ~0)
 165         return 1;
 166
 167       nat44_ei_interface_t *i;
 168       pool_foreach (i, nm->interfaces)
 169         {
 170           /* NAT packet aimed at outside interface */
 171           if ((nat44_ei_interface_is_outside (i)) &&
 172               (sw_if_index == i->sw_if_index))
 173             return 0;
 174         }
 175     }
 176
 177   return 1;
 178 }
 179
 180 static inline int
 181 nat44_ei_not_translate (nat44_ei_main_t *nm, vlib_node_runtime_t *node,
 182                         u32 sw_if_index0, ip4_header_t *ip0, u32 proto0,
 183                         u32 rx_fib_index0, u32 thread_index)
 184 {
 185   udp_header_t *udp0 = ip4_next_header (ip0);
 186   clib_bihash_kv_8_8_t kv0, value0;
 187
 188   init_nat_k (&kv0, ip0->dst_address, udp0->dst_port, nm->outside_fib_index,
 189               proto0);
 190
 191   /* NAT packet aimed at external address if */
 192   /* has active sessions */
 193   if (clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
 194     {
 195       /* or is static mappings */
 196       ip4_address_t placeholder_addr;
 197       u16 placeholder_port;
 198       u32 placeholder_fib_index;
 199       if (!nat44_ei_static_mapping_match (ip0->dst_address, udp0->dst_port,
 200                                           nm->outside_fib_index, proto0,
 201                                           &placeholder_addr, &placeholder_port,
 202                                           &placeholder_fib_index, 1, 0, 0))
 203         return 0;
 204     }
 205   else
 206     return 0;
 207
 208   if (nm->forwarding_enabled)
 209     return 1;
 210
 211   return nat44_ei_not_translate_fast (node, sw_if_index0, ip0, proto0,
 212                                       rx_fib_index0);
 213 }
 214
 215 static inline int
 216 nat44_ei_not_translate_output_feature (nat44_ei_main_t *nm, ip4_header_t *ip0,
 217                                        u32 proto0, u16 src_port, u16 dst_port,
 218                                        u32 thread_index, u32 sw_if_index)
 219 {
 220   clib_bihash_kv_8_8_t kv0, value0;
 221   nat44_ei_interface_t *i;
 222
 223   /* src NAT check */
 224   init_nat_k (&kv0, ip0->src_address, src_port,
 225               ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
 226
 227   if (!clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
 228     return 1;
 229
 230   /* dst NAT check */
 231   init_nat_k (&kv0, ip0->dst_address, dst_port,
 232               ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
 233   if (!clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
 234     {
 235       /* hairpinning */
 236       pool_foreach (i, nm->output_feature_interfaces)
 237         {
 238           if ((nat44_ei_interface_is_inside (i)) &&
 239               (sw_if_index == i->sw_if_index))
 240             return 0;
 241         }
 242       return 1;
 243     }
 244
 245   return 0;
 246 }
 247
 248 #ifndef CLIB_MARCH_VARIANT
 249 int
 250 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 251 {
 252   nat44_ei_main_t *nm = &nat44_ei_main;
 253   nat44_ei_is_idle_session_ctx_t *ctx = arg;
 254   nat44_ei_session_t *s;
 255   u64 sess_timeout_time;
 256   nat44_ei_main_per_thread_data_t *tnm =
 257     vec_elt_at_index (nm->per_thread_data, ctx->thread_index);
 258   clib_bihash_kv_8_8_t s_kv;
 259
 260   if (ctx->thread_index != nat_value_get_thread_index (kv))
 261     {
 262       return 0;
 263     }
 264
 265   s = pool_elt_at_index (tnm->sessions, nat_value_get_session_index (kv));
 266   sess_timeout_time = s->last_heard + (f64) nat_session_get_timeout (
 267                                         &nm->timeouts, s->nat_proto, s->state);
 268   if (ctx->now >= sess_timeout_time)
 269     {
 270       init_nat_o2i_k (&s_kv, s);
 271       if (clib_bihash_add_del_8_8 (&nm->out2in, &s_kv, 0))
 272         nat_elog_warn (nm, "out2in key del failed");
 273
 274       nat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 275                                           s->in2out.addr.as_u32,
 276                                           s->out2in.addr.as_u32,
 277                                           s->nat_proto,
 278                                           s->in2out.port,
 279                                           s->out2in.port,
 280                                           s->in2out.fib_index);
 281
 282       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 283                                &s->in2out.addr, s->in2out.port,
 284                                &s->out2in.addr, s->out2in.port, s->nat_proto);
 285
 286       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 287                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 288                    ctx->thread_index);
 289
 290       if (!nat44_ei_is_session_static (s))
 291         nat44_ei_free_outside_address_and_port (
 292           nm->addresses, ctx->thread_index, &s->out2in.addr, s->out2in.port,
 293           s->nat_proto);
 294
 295       nat44_ei_delete_session (nm, s, ctx->thread_index);
 296       return 1;
 297     }
 298
 299   return 0;
 300 }
 301 #endif
 302
 303 static u32
 304 slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0, ip4_header_t *ip0,
 305            ip4_address_t i2o_addr, u16 i2o_port, u32 rx_fib_index0,
 306            nat_protocol_t nat_proto, nat44_ei_session_t **sessionp,
 307            vlib_node_runtime_t *node, u32 next0, u32 thread_index, f64 now)
 308 {
 309   nat44_ei_user_t *u;
 310   nat44_ei_session_t *s = 0;
 311   clib_bihash_kv_8_8_t kv0;
 312   u8 is_sm = 0;
 313   nat44_ei_outside_fib_t *outside_fib;
 314   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 315   u8 identity_nat;
 316   fib_prefix_t pfx = {
 317     .fp_proto = FIB_PROTOCOL_IP4,
 318     .fp_len = 32,
 319     .fp_addr = {
 320                 .ip4.as_u32 = ip0->dst_address.as_u32,
 321                 },
 322   };
 323   nat44_ei_is_idle_session_ctx_t ctx0;
 324   ip4_address_t sm_addr;
 325   u16 sm_port;
 326   u32 sm_fib_index;
 327
 328   if (PREDICT_FALSE (nat44_ei_maximum_sessions_exceeded (nm, thread_index)))
 329     {
 330       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 331       nat_ipfix_logging_max_sessions (thread_index,
 332                                       nm->max_translations_per_thread);
 333       nat_elog_notice (nm, "maximum sessions exceeded");
 334       return NAT44_EI_IN2OUT_NEXT_DROP;
 335     }
 336
 337   /* First try to match static mapping by local address and port */
 338   if (nat44_ei_static_mapping_match (i2o_addr, i2o_port, rx_fib_index0,
 339                                      nat_proto, &sm_addr, &sm_port,
 340                                      &sm_fib_index, 0, 0, &identity_nat))
 341     {
 342       /* Try to create dynamic translation */
 343       if (nm->alloc_addr_and_port (
 344             nm->addresses, rx_fib_index0, thread_index, nat_proto,
 345             ip0->src_address, &sm_addr, &sm_port, nm->port_per_thread,
 346             nm->per_thread_data[thread_index].snat_thread_index))
 347         {
 348           b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_OUT_OF_PORTS];
 349           return NAT44_EI_IN2OUT_NEXT_DROP;
 350         }
 351     }
 352   else
 353     {
 354       if (PREDICT_FALSE (identity_nat))
 355         {
 356           *sessionp = s;
 357           return next0;
 358         }
 359
 360       is_sm = 1;
 361     }
 362
 363   u = nat44_ei_user_get_or_create (nm, &ip0->src_address, rx_fib_index0,
 364                                    thread_index);
 365   if (!u)
 366     {
 367       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_CANNOT_CREATE_USER];
 368       return NAT44_EI_IN2OUT_NEXT_DROP;
 369     }
 370
 371   s = nat44_ei_session_alloc_or_recycle (nm, u, thread_index, now);
 372   if (!s)
 373     {
 374       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
 375       nat_elog_warn (nm, "create NAT session failed");
 376       return NAT44_EI_IN2OUT_NEXT_DROP;
 377     }
 378
 379   if (is_sm)
 380     s->flags |= NAT44_EI_SESSION_FLAG_STATIC_MAPPING;
 381   nat44_ei_user_session_increment (nm, u, is_sm);
 382   s->in2out.addr = i2o_addr;
 383   s->in2out.port = i2o_port;
 384   s->in2out.fib_index = rx_fib_index0;
 385   s->nat_proto = nat_proto;
 386   s->out2in.addr = sm_addr;
 387   s->out2in.port = sm_port;
 388   s->out2in.fib_index = nm->outside_fib_index;
 389   switch (vec_len (nm->outside_fibs))
 390     {
 391     case 0:
 392       s->out2in.fib_index = nm->outside_fib_index;
 393       break;
 394     case 1:
 395       s->out2in.fib_index = nm->outside_fibs[0].fib_index;
 396       break;
 397     default:
 398       vec_foreach (outside_fib, nm->outside_fibs)
 399         {
 400           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 401           if (FIB_NODE_INDEX_INVALID != fei)
 402             {
 403               if (fib_entry_get_resolving_interface (fei) != ~0)
 404                 {
 405                   s->out2in.fib_index = outside_fib->fib_index;
 406                   break;
 407                 }
 408             }
 409         }
 410       break;
 411     }
 412   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 413   s->ext_host_port = vnet_buffer (b0)->ip.reass.l4_dst_port;
 414   *sessionp = s;
 415
 416   /* Add to translation hashes */
 417   ctx0.now = now;
 418   ctx0.thread_index = thread_index;
 419   init_nat_i2o_kv (&kv0, s, thread_index,
 420                    s - nm->per_thread_data[thread_index].sessions);
 421   if (clib_bihash_add_or_overwrite_stale_8_8 (
 422         &nm->in2out, &kv0, nat44_i2o_is_idle_session_cb, &ctx0))
 423     nat_elog_notice (nm, "in2out key add failed");
 424
 425   init_nat_o2i_kv (&kv0, s, thread_index,
 426                    s - nm->per_thread_data[thread_index].sessions);
 427   if (clib_bihash_add_or_overwrite_stale_8_8 (
 428         &nm->out2in, &kv0, nat44_o2i_is_idle_session_cb, &ctx0))
 429     nat_elog_notice (nm, "out2in key add failed");
 430
 431   /* log NAT event */
 432   nat_ipfix_logging_nat44_ses_create (
 433     thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->nat_proto,
 434     s->in2out.port, s->out2in.port, s->in2out.fib_index);
 435
 436   nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index, &s->in2out.addr,
 437                            s->in2out.port, &s->out2in.addr, s->out2in.port,
 438                            s->nat_proto);
 439
 440   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 441                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 442                &s->ext_host_nat_addr, s->ext_host_nat_port, s->nat_proto,
 443                s->in2out.fib_index, s->flags, thread_index, 0);
 444
 445   return next0;
 446 }
 447
 448 #ifndef CLIB_MARCH_VARIANT
 449 static_always_inline nat44_ei_in2out_error_t
 450 icmp_get_key (vlib_buffer_t *b, ip4_header_t *ip0, ip4_address_t *addr,
 451               u16 *port, nat_protocol_t *nat_proto)
 452 {
 453   icmp46_header_t *icmp0;
 454   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 455   ip4_header_t *inner_ip0 = 0;
 456   void *l4_header = 0;
 457   icmp46_header_t *inner_icmp0;
 458
 459   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 460   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 461
 462   if (!icmp_type_is_error_message
 463       (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 464     {
 465       *nat_proto = NAT_PROTOCOL_ICMP;
 466       *addr = ip0->src_address;
 467       *port = vnet_buffer (b)->ip.reass.l4_src_port;
 468     }
 469   else
 470     {
 471       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 472       l4_header = ip4_next_header (inner_ip0);
 473       *nat_proto = ip_proto_to_nat_proto (inner_ip0->protocol);
 474       *addr = inner_ip0->dst_address;
 475       switch (*nat_proto)
 476         {
 477         case NAT_PROTOCOL_ICMP:
 478           inner_icmp0 = (icmp46_header_t *) l4_header;
 479           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 480           *port = inner_echo0->identifier;
 481           break;
 482         case NAT_PROTOCOL_UDP:
 483         case NAT_PROTOCOL_TCP:
 484           *port = ((tcp_udp_header_t *) l4_header)->dst_port;
 485           break;
 486         default:
 487           return NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 488         }
 489     }
 490   return -1;                    /* success */
 491 }
 492
 493 /**
 494  * Get address and port values to be used for ICMP packet translation
 495  * and create session if needed
 496  *
 497  * @param[in,out] nm             NAT main
 498  * @param[in,out] node           NAT node runtime
 499  * @param[in] thread_index       thread index
 500  * @param[in,out] b0             buffer containing packet to be translated
 501  * @param[in,out] ip0            ip header
 502  * @param[out] p_proto           protocol used for matching
 503  * @param[out] p_value           address and port after NAT translation
 504  * @param[out] p_dont_translate  if packet should not be translated
 505  * @param d                      optional parameter
 506  * @param e                      optional parameter
 507  */
 508 u32
 509 nat44_ei_icmp_match_in2out_slow (vlib_node_runtime_t *node, u32 thread_index,
 510                                  vlib_buffer_t *b0, ip4_header_t *ip0,
 511                                  ip4_address_t *addr, u16 *port,
 512                                  u32 *fib_index, nat_protocol_t *proto,
 513                                  nat44_ei_session_t **p_s0, u8 *dont_translate)
 514 {
 515   nat44_ei_main_t *nm = &nat44_ei_main;
 516   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
 517   u32 sw_if_index0;
 518   nat44_ei_session_t *s0 = 0;
 519   clib_bihash_kv_8_8_t kv0, value0;
 520   u32 next0 = ~0;
 521   int err;
 522   vlib_main_t *vm = vlib_get_main ();
 523   *dont_translate = 0;
 524
 525   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 526   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 527
 528   err = icmp_get_key (b0, ip0, addr, port, proto);
 529   if (err != -1)
 530     {
 531       b0->error = node->errors[err];
 532       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 533       goto out;
 534     }
 535
 536   init_nat_k (&kv0, *addr, *port, *fib_index, *proto);
 537   if (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
 538     {
 539       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 540         {
 541           if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
 542                 nm, ip0, *proto, *port, *port, thread_index, sw_if_index0)))
 543             {
 544               *dont_translate = 1;
 545               goto out;
 546             }
 547         }
 548       else
 549         {
 550           if (PREDICT_FALSE (nat44_ei_not_translate (
 551                 nm, node, sw_if_index0, ip0, NAT_PROTOCOL_ICMP, *fib_index,
 552                 thread_index)))
 553             {
 554               *dont_translate = 1;
 555               goto out;
 556             }
 557         }
 558
 559       if (PREDICT_FALSE
 560           (icmp_type_is_error_message
 561            (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags)))
 562         {
 563           b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
 564           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 565           goto out;
 566         }
 567
 568       next0 = slow_path (nm, b0, ip0, *addr, *port, *fib_index, *proto, &s0,
 569                          node, next0, thread_index, vlib_time_now (vm));
 570
 571       if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
 572         goto out;
 573
 574       if (!s0)
 575         {
 576           *dont_translate = 1;
 577           goto out;
 578         }
 579     }
 580   else
 581     {
 582       if (PREDICT_FALSE
 583           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 584            ICMP4_echo_request
 585            && vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 586            ICMP4_echo_reply
 587            && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 588                                            reass.icmp_type_or_tcp_flags)))
 589         {
 590           b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
 591           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 592           goto out;
 593         }
 594
 595       s0 = pool_elt_at_index (tnm->sessions,
 596                               nat_value_get_session_index (&value0));
 597     }
 598
 599 out:
 600   if (s0)
 601     {
 602       *addr = s0->out2in.addr;
 603       *port = s0->out2in.port;
 604       *fib_index = s0->out2in.fib_index;
 605     }
 606   if (p_s0)
 607     *p_s0 = s0;
 608   return next0;
 609 }
 610 #endif
 611
 612 #ifndef CLIB_MARCH_VARIANT
 613 u32
 614 nat44_ei_icmp_match_in2out_fast (vlib_node_runtime_t *node, u32 thread_index,
 615                                  vlib_buffer_t *b0, ip4_header_t *ip0,
 616                                  ip4_address_t *addr, u16 *port,
 617                                  u32 *fib_index, nat_protocol_t *proto,
 618                                  nat44_ei_session_t **s0, u8 *dont_translate)
 619 {
 620   u32 sw_if_index0;
 621   u8 is_addr_only;
 622   u32 next0 = ~0;
 623   int err;
 624   *dont_translate = 0;
 625
 626   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 627   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 628
 629   err = icmp_get_key (b0, ip0, addr, port, proto);
 630   if (err != -1)
 631     {
 632       b0->error = node->errors[err];
 633       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 634       goto out;
 635     }
 636
 637   ip4_address_t sm_addr;
 638   u16 sm_port;
 639   u32 sm_fib_index;
 640
 641   if (nat44_ei_static_mapping_match (*addr, *port, *fib_index, *proto,
 642                                      &sm_addr, &sm_port, &sm_fib_index, 0,
 643                                      &is_addr_only, 0))
 644     {
 645       if (PREDICT_FALSE (nat44_ei_not_translate_fast (
 646             node, sw_if_index0, ip0, IP_PROTOCOL_ICMP, *fib_index)))
 647         {
 648           *dont_translate = 1;
 649           goto out;
 650         }
 651
 652       if (icmp_type_is_error_message
 653           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 654         {
 655           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 656           goto out;
 657         }
 658
 659       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_NO_TRANSLATION];
 660       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 661       goto out;
 662     }
 663
 664   if (PREDICT_FALSE
 665       (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags != ICMP4_echo_request
 666        && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 667            ICMP4_echo_reply || !is_addr_only)
 668        && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 669                                        reass.icmp_type_or_tcp_flags)))
 670     {
 671       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
 672       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 673       goto out;
 674     }
 675
 676 out:
 677   return next0;
 678 }
 679 #endif
 680
 681 u32 nat44_ei_icmp_in2out (vlib_buffer_t *b0, ip4_header_t *ip0,
 682                           icmp46_header_t *icmp0, u32 sw_if_index0,
 683                           u32 rx_fib_index0, vlib_node_runtime_t *node,
 684                           u32 next0, u32 thread_index,
 685                           nat44_ei_session_t **p_s0);
 686
 687 #ifndef CLIB_MARCH_VARIANT
 688 u32
 689 nat44_ei_icmp_in2out (vlib_buffer_t *b0, ip4_header_t *ip0,
 690                       icmp46_header_t *icmp0, u32 sw_if_index0,
 691                       u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0,
 692                       u32 thread_index, nat44_ei_session_t **p_s0)
 693 {
 694   nat44_ei_main_t *nm = &nat44_ei_main;
 695   vlib_main_t *vm = vlib_get_main ();
 696   ip4_address_t addr;
 697   u16 port;
 698   u32 fib_index;
 699   nat_protocol_t proto;
 700   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 701   ip4_header_t *inner_ip0;
 702   void *l4_header = 0;
 703   icmp46_header_t *inner_icmp0;
 704   u8 dont_translate;
 705   u32 new_addr0, old_addr0;
 706   u16 old_id0, new_id0;
 707   u16 old_checksum0, new_checksum0;
 708   ip_csum_t sum0;
 709   u16 checksum0;
 710   u32 next0_tmp;
 711   u32 required_thread_index = thread_index;
 712
 713   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 714
 715   if (PREDICT_TRUE (nm->pat))
 716     {
 717       next0_tmp = nat44_ei_icmp_match_in2out_slow (
 718         node, thread_index, b0, ip0, &addr, &port, &fib_index, &proto, p_s0,
 719         &dont_translate);
 720     }
 721   else
 722     {
 723       next0_tmp = nat44_ei_icmp_match_in2out_fast (
 724         node, thread_index, b0, ip0, &addr, &port, &fib_index, &proto, p_s0,
 725         &dont_translate);
 726     }
 727
 728   if (next0_tmp != ~0)
 729     next0 = next0_tmp;
 730   if (next0 == NAT44_EI_IN2OUT_NEXT_DROP || dont_translate)
 731     goto out;
 732
 733   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 734     {
 735       sum0 =
 736         ip_incremental_checksum_buffer (vm, b0,
 737                                         (u8 *) icmp0 -
 738                                         (u8 *) vlib_buffer_get_current (b0),
 739                                         ntohs (ip0->length) -
 740                                         ip4_header_bytes (ip0), 0);
 741       checksum0 = ~ip_csum_fold (sum0);
 742       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 743         {
 744           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 745           goto out;
 746         }
 747     }
 748
 749   old_addr0 = ip0->src_address.as_u32;
 750   new_addr0 = ip0->src_address.as_u32 = addr.as_u32;
 751
 752   sum0 = ip0->checksum;
 753   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 754                          src_address /* changed member */ );
 755   ip0->checksum = ip_csum_fold (sum0);
 756
 757   if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 758     {
 759       if (icmp0->checksum == 0)
 760         icmp0->checksum = 0xffff;
 761
 762       if (!icmp_type_is_error_message (icmp0->type))
 763         {
 764           new_id0 = port;
 765           if (PREDICT_FALSE (new_id0 != echo0->identifier))
 766             {
 767               old_id0 = echo0->identifier;
 768               new_id0 = port;
 769               echo0->identifier = new_id0;
 770
 771               sum0 = icmp0->checksum;
 772               sum0 =
 773                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 774                                 identifier);
 775               icmp0->checksum = ip_csum_fold (sum0);
 776             }
 777         }
 778       else
 779         {
 780           inner_ip0 = (ip4_header_t *) (echo0 + 1);
 781           l4_header = ip4_next_header (inner_ip0);
 782
 783           if (!ip4_header_checksum_is_valid (inner_ip0))
 784             {
 785               next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 786               goto out;
 787             }
 788
 789           /* update inner destination IP address */
 790           old_addr0 = inner_ip0->dst_address.as_u32;
 791           inner_ip0->dst_address = addr;
 792           new_addr0 = inner_ip0->dst_address.as_u32;
 793           sum0 = icmp0->checksum;
 794           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 795                                  dst_address /* changed member */ );
 796           icmp0->checksum = ip_csum_fold (sum0);
 797
 798           /* update inner IP header checksum */
 799           old_checksum0 = inner_ip0->checksum;
 800           sum0 = inner_ip0->checksum;
 801           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 802                                  dst_address /* changed member */ );
 803           inner_ip0->checksum = ip_csum_fold (sum0);
 804           new_checksum0 = inner_ip0->checksum;
 805           sum0 = icmp0->checksum;
 806           sum0 =
 807             ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 808                             checksum);
 809           icmp0->checksum = ip_csum_fold (sum0);
 810
 811           switch (proto)
 812             {
 813             case NAT_PROTOCOL_ICMP:
 814               inner_icmp0 = (icmp46_header_t *) l4_header;
 815               inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 816
 817               old_id0 = inner_echo0->identifier;
 818               new_id0 = port;
 819               inner_echo0->identifier = new_id0;
 820
 821               sum0 = icmp0->checksum;
 822               sum0 =
 823                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 824                                 identifier);
 825               icmp0->checksum = ip_csum_fold (sum0);
 826               break;
 827             case NAT_PROTOCOL_UDP:
 828             case NAT_PROTOCOL_TCP:
 829               old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 830               new_id0 = port;
 831               ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 832
 833               sum0 = icmp0->checksum;
 834               sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 835                                      dst_port);
 836               icmp0->checksum = ip_csum_fold (sum0);
 837               break;
 838             default:
 839               ASSERT (0);
 840             }
 841         }
 842     }
 843
 844   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 845     {
 846       if (0 != nat44_ei_icmp_hairpinning (nm, b0, thread_index, ip0, icmp0,
 847                                           &required_thread_index))
 848         vnet_buffer (b0)->sw_if_index[VLIB_TX] = fib_index;
 849       if (thread_index != required_thread_index)
 850         {
 851           vnet_buffer (b0)->snat.required_thread_index = required_thread_index;
 852           next0 = NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF;
 853         }
 854     }
 855
 856 out:
 857   return next0;
 858 }
 859 #endif
 860
 861 static_always_inline u32
 862 nat44_ei_icmp_in2out_slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0,
 863                                 ip4_header_t *ip0, icmp46_header_t *icmp0,
 864                                 u32 sw_if_index0, u32 rx_fib_index0,
 865                                 vlib_node_runtime_t *node, u32 next0, f64 now,
 866                                 u32 thread_index, nat44_ei_session_t **p_s0)
 867 {
 868   vlib_main_t *vm = vlib_get_main ();
 869
 870   next0 = nat44_ei_icmp_in2out (b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 871                                 node, next0, thread_index, p_s0);
 872   nat44_ei_session_t *s0 = *p_s0;
 873   if (PREDICT_TRUE (next0 != NAT44_EI_IN2OUT_NEXT_DROP && s0))
 874     {
 875       /* Accounting */
 876       nat44_ei_session_update_counters (
 877         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
 878       /* Per-user LRU list maintenance */
 879       nat44_ei_session_update_lru (nm, s0, thread_index);
 880     }
 881   return next0;
 882 }
 883
 884 static int
 885 nat_in2out_sm_unknown_proto (nat44_ei_main_t *nm, vlib_buffer_t *b,
 886                              ip4_header_t *ip, u32 rx_fib_index)
 887 {
 888   clib_bihash_kv_8_8_t kv, value;
 889   nat44_ei_static_mapping_t *m;
 890   u32 old_addr, new_addr;
 891   ip_csum_t sum;
 892
 893   init_nat_k (&kv, ip->src_address, 0, rx_fib_index, 0);
 894   if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
 895     return 1;
 896
 897   m = pool_elt_at_index (nm->static_mappings, value.value);
 898
 899   old_addr = ip->src_address.as_u32;
 900   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 901   sum = ip->checksum;
 902   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 903   ip->checksum = ip_csum_fold (sum);
 904
 905
 906   /* Hairpinning */
 907   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 908     {
 909       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 910       nat44_ei_hairpinning_sm_unknown_proto (nm, b, ip);
 911     }
 912
 913   return 0;
 914 }
 915
 916 static inline uword
 917 nat44_ei_in2out_node_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
 918                                 vlib_frame_t *frame, int is_slow_path,
 919                                 int is_output_feature)
 920 {
 921   u32 n_left_from, *from;
 922   nat44_ei_main_t *nm = &nat44_ei_main;
 923   f64 now = vlib_time_now (vm);
 924   u32 thread_index = vm->thread_index;
 925
 926   from = vlib_frame_vector_args (frame);
 927   n_left_from = frame->n_vectors;
 928
 929   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 930   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 931   vlib_get_buffers (vm, from, b, n_left_from);
 932
 933   while (n_left_from >= 2)
 934     {
 935       vlib_buffer_t *b0, *b1;
 936       u32 next0, next1;
 937       u32 rx_sw_if_index0, rx_sw_if_index1;
 938       u32 tx_sw_if_index0, tx_sw_if_index1;
 939       u32 cntr_sw_if_index0, cntr_sw_if_index1;
 940       ip4_header_t *ip0, *ip1;
 941       ip_csum_t sum0, sum1;
 942       u32 new_addr0, old_addr0, new_addr1, old_addr1;
 943       u16 old_port0, new_port0, old_port1, new_port1;
 944       udp_header_t *udp0, *udp1;
 945       tcp_header_t *tcp0, *tcp1;
 946       icmp46_header_t *icmp0, *icmp1;
 947       u32 rx_fib_index0, rx_fib_index1;
 948       u32 proto0, proto1;
 949       nat44_ei_session_t *s0 = 0, *s1 = 0;
 950       clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 951       u32 iph_offset0 = 0, iph_offset1 = 0;
 952
 953       b0 = *b;
 954       b++;
 955       b1 = *b;
 956       b++;
 957
 958       /* Prefetch next iteration. */
 959       if (PREDICT_TRUE (n_left_from >= 4))
 960         {
 961           vlib_buffer_t *p2, *p3;
 962
 963           p2 = *b;
 964           p3 = *(b + 1);
 965
 966           vlib_prefetch_buffer_header (p2, LOAD);
 967           vlib_prefetch_buffer_header (p3, LOAD);
 968
 969           clib_prefetch_load (p2->data);
 970           clib_prefetch_load (p3->data);
 971         }
 972
 973       if (is_output_feature)
 974         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 975
 976       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 977                               iph_offset0);
 978
 979       udp0 = ip4_next_header (ip0);
 980       tcp0 = (tcp_header_t *) udp0;
 981       icmp0 = (icmp46_header_t *) udp0;
 982
 983       rx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 984       tx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
 985       cntr_sw_if_index0 =
 986         is_output_feature ? tx_sw_if_index0 : rx_sw_if_index0;
 987       rx_fib_index0 =
 988         vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index0);
 989
 990       next0 = next1 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
 991
 992       if (PREDICT_FALSE (ip0->ttl == 1))
 993         {
 994           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 995           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 996                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 997                                        0);
 998           next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
 999           goto trace00;
1000         }
1001
1002       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1003
1004       /* Next configured feature, probably ip4-lookup */
1005       if (is_slow_path)
1006         {
1007           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1008             {
1009               if (nat_in2out_sm_unknown_proto (nm, b0, ip0, rx_fib_index0))
1010                 {
1011                   next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1012                   b0->error =
1013                     node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1014                 }
1015               vlib_increment_simple_counter (
1016                 is_slow_path ? &nm->counters.slowpath.in2out.other :
1017                                &nm->counters.fastpath.in2out.other,
1018                 thread_index, cntr_sw_if_index0, 1);
1019               goto trace00;
1020             }
1021
1022           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1023             {
1024               next0 = nat44_ei_icmp_in2out_slow_path (
1025                 nm, b0, ip0, icmp0, rx_sw_if_index0, rx_fib_index0, node,
1026                 next0, now, thread_index, &s0);
1027               vlib_increment_simple_counter (
1028                 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1029                                &nm->counters.fastpath.in2out.icmp,
1030                 thread_index, cntr_sw_if_index0, 1);
1031               goto trace00;
1032             }
1033         }
1034       else
1035         {
1036           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1037             {
1038               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1039               goto trace00;
1040             }
1041
1042           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1043             {
1044               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1045               goto trace00;
1046             }
1047         }
1048
1049       init_nat_k (&kv0, ip0->src_address,
1050                   vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1051                   proto0);
1052       if (PREDICT_FALSE (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0) !=
1053                          0))
1054         {
1055           if (is_slow_path)
1056             {
1057               if (is_output_feature)
1058                 {
1059                   if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1060                         nm, ip0, proto0,
1061                         vnet_buffer (b0)->ip.reass.l4_src_port,
1062                         vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1063                         rx_sw_if_index0)))
1064                     goto trace00;
1065
1066                   /*
1067                    * Send DHCP packets to the ipv4 stack, or we won't
1068                    * be able to use dhcp client on the outside interface
1069                    */
1070                   if (PREDICT_FALSE
1071                       (proto0 == NAT_PROTOCOL_UDP
1072                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1073                            clib_host_to_net_u16
1074                            (UDP_DST_PORT_dhcp_to_server))
1075                        && ip0->dst_address.as_u32 == 0xffffffff))
1076                     goto trace00;
1077                 }
1078               else
1079                 {
1080                   if (PREDICT_FALSE (nat44_ei_not_translate (
1081                         nm, node, rx_sw_if_index0, ip0, proto0, rx_fib_index0,
1082                         thread_index)))
1083                     goto trace00;
1084                 }
1085
1086               next0 = slow_path (nm, b0, ip0, ip0->src_address,
1087                                  vnet_buffer (b0)->ip.reass.l4_src_port,
1088                                  rx_fib_index0, proto0, &s0, node, next0,
1089                                  thread_index, now);
1090               if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
1091                 goto trace00;
1092
1093               if (PREDICT_FALSE (!s0))
1094                 goto trace00;
1095             }
1096           else
1097             {
1098               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1099               goto trace00;
1100             }
1101         }
1102       else
1103         s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1104                                 nat_value_get_session_index (&value0));
1105
1106       b0->flags |= VNET_BUFFER_F_IS_NATED;
1107
1108       old_addr0 = ip0->src_address.as_u32;
1109       ip0->src_address = s0->out2in.addr;
1110       new_addr0 = ip0->src_address.as_u32;
1111       if (!is_output_feature)
1112         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1113
1114       sum0 = ip0->checksum;
1115       sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1116                              ip4_header_t, src_address /* changed member */ );
1117       ip0->checksum = ip_csum_fold (sum0);
1118
1119
1120       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1121         {
1122           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1123             {
1124               old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1125               new_port0 = udp0->src_port = s0->out2in.port;
1126               sum0 = tcp0->checksum;
1127               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1128                                      ip4_header_t,
1129                                      dst_address /* changed member */ );
1130               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1131                                      ip4_header_t /* cheat */ ,
1132                                      length /* changed member */ );
1133               mss_clamping (nm->mss_clamping, tcp0, &sum0);
1134               tcp0->checksum = ip_csum_fold (sum0);
1135             }
1136           vlib_increment_simple_counter (is_slow_path ?
1137                                            &nm->counters.slowpath.in2out.tcp :
1138                                            &nm->counters.fastpath.in2out.tcp,
1139                                          thread_index, cntr_sw_if_index0, 1);
1140         }
1141       else
1142         {
1143           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1144             {
1145               udp0->src_port = s0->out2in.port;
1146               if (PREDICT_FALSE (udp0->checksum))
1147                 {
1148                   old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1149                   new_port0 = udp0->src_port;
1150                   sum0 = udp0->checksum;
1151                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address  /* changed member */
1152                     );
1153                   sum0 =
1154                     ip_csum_update (sum0, old_port0, new_port0,
1155                                     ip4_header_t /* cheat */ ,
1156                                     length /* changed member */ );
1157                   udp0->checksum = ip_csum_fold (sum0);
1158                 }
1159             }
1160           vlib_increment_simple_counter (is_slow_path ?
1161                                            &nm->counters.slowpath.in2out.udp :
1162                                            &nm->counters.fastpath.in2out.udp,
1163                                          thread_index, cntr_sw_if_index0, 1);
1164         }
1165
1166       /* Accounting */
1167       nat44_ei_session_update_counters (
1168         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1169       /* Per-user LRU list maintenance */
1170       nat44_ei_session_update_lru (nm, s0, thread_index);
1171     trace00:
1172
1173       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1174                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1175         {
1176           nat44_ei_in2out_trace_t *t =
1177             vlib_add_trace (vm, node, b0, sizeof (*t));
1178           t->is_slow_path = is_slow_path;
1179           t->sw_if_index = rx_sw_if_index0;
1180           t->next_index = next0;
1181           t->session_index = ~0;
1182           if (s0)
1183             t->session_index = s0 - nm->per_thread_data[thread_index].sessions;
1184         }
1185
1186       if (next0 == NAT44_EI_IN2OUT_NEXT_DROP)
1187         {
1188           vlib_increment_simple_counter (
1189             is_slow_path ? &nm->counters.slowpath.in2out.drops :
1190                            &nm->counters.fastpath.in2out.drops,
1191             thread_index, cntr_sw_if_index0, 1);
1192         }
1193
1194       if (is_output_feature)
1195         iph_offset1 = vnet_buffer (b1)->ip.reass.save_rewrite_length;
1196
1197       ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1198                               iph_offset1);
1199
1200       udp1 = ip4_next_header (ip1);
1201       tcp1 = (tcp_header_t *) udp1;
1202       icmp1 = (icmp46_header_t *) udp1;
1203
1204       rx_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1205       tx_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_TX];
1206       cntr_sw_if_index1 =
1207         is_output_feature ? tx_sw_if_index1 : rx_sw_if_index1;
1208       rx_fib_index1 =
1209         vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index1);
1210
1211       if (PREDICT_FALSE (ip1->ttl == 1))
1212         {
1213           vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1214           icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1215                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1216                                        0);
1217           next1 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1218           goto trace01;
1219         }
1220
1221       proto1 = ip_proto_to_nat_proto (ip1->protocol);
1222
1223       /* Next configured feature, probably ip4-lookup */
1224       if (is_slow_path)
1225         {
1226           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1227             {
1228               if (nat_in2out_sm_unknown_proto (nm, b1, ip1, rx_fib_index1))
1229                 {
1230                   next1 = NAT44_EI_IN2OUT_NEXT_DROP;
1231                   b1->error =
1232                     node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1233                 }
1234               vlib_increment_simple_counter (
1235                 is_slow_path ? &nm->counters.slowpath.in2out.other :
1236                                &nm->counters.fastpath.in2out.other,
1237                 thread_index, cntr_sw_if_index1, 1);
1238               goto trace01;
1239             }
1240
1241           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1242             {
1243               next1 = nat44_ei_icmp_in2out_slow_path (
1244                 nm, b1, ip1, icmp1, rx_sw_if_index1, rx_fib_index1, node,
1245                 next1, now, thread_index, &s1);
1246               vlib_increment_simple_counter (
1247                 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1248                                &nm->counters.fastpath.in2out.icmp,
1249                 thread_index, cntr_sw_if_index1, 1);
1250               goto trace01;
1251             }
1252         }
1253       else
1254         {
1255           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1256             {
1257               next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1258               goto trace01;
1259             }
1260
1261           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1262             {
1263               next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1264               goto trace01;
1265             }
1266         }
1267
1268       init_nat_k (&kv1, ip1->src_address,
1269                   vnet_buffer (b1)->ip.reass.l4_src_port, rx_fib_index1,
1270                   proto1);
1271       if (PREDICT_FALSE (clib_bihash_search_8_8 (&nm->in2out, &kv1, &value1) !=
1272                          0))
1273         {
1274           if (is_slow_path)
1275             {
1276               if (is_output_feature)
1277                 {
1278                   if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1279                         nm, ip1, proto1,
1280                         vnet_buffer (b1)->ip.reass.l4_src_port,
1281                         vnet_buffer (b1)->ip.reass.l4_dst_port, thread_index,
1282                         rx_sw_if_index1)))
1283                     goto trace01;
1284
1285                   /*
1286                    * Send DHCP packets to the ipv4 stack, or we won't
1287                    * be able to use dhcp client on the outside interface
1288                    */
1289                   if (PREDICT_FALSE
1290                       (proto1 == NAT_PROTOCOL_UDP
1291                        && (vnet_buffer (b1)->ip.reass.l4_dst_port ==
1292                            clib_host_to_net_u16
1293                            (UDP_DST_PORT_dhcp_to_server))
1294                        && ip1->dst_address.as_u32 == 0xffffffff))
1295                     goto trace01;
1296                 }
1297               else
1298                 {
1299                   if (PREDICT_FALSE (nat44_ei_not_translate (
1300                         nm, node, rx_sw_if_index1, ip1, proto1, rx_fib_index1,
1301                         thread_index)))
1302                     goto trace01;
1303                 }
1304
1305               next1 = slow_path (nm, b1, ip1, ip1->src_address,
1306                                  vnet_buffer (b1)->ip.reass.l4_src_port,
1307                                  rx_fib_index1, proto1, &s1, node, next1,
1308                                  thread_index, now);
1309               if (PREDICT_FALSE (next1 == NAT44_EI_IN2OUT_NEXT_DROP))
1310                 goto trace01;
1311
1312               if (PREDICT_FALSE (!s1))
1313                 goto trace01;
1314             }
1315           else
1316             {
1317               next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1318               goto trace01;
1319             }
1320         }
1321       else
1322         s1 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1323                                 nat_value_get_session_index (&value1));
1324
1325       b1->flags |= VNET_BUFFER_F_IS_NATED;
1326
1327       old_addr1 = ip1->src_address.as_u32;
1328       ip1->src_address = s1->out2in.addr;
1329       new_addr1 = ip1->src_address.as_u32;
1330       if (!is_output_feature)
1331         vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1332
1333       sum1 = ip1->checksum;
1334       sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1335                              ip4_header_t, src_address /* changed member */ );
1336       ip1->checksum = ip_csum_fold (sum1);
1337
1338       if (PREDICT_TRUE (proto1 == NAT_PROTOCOL_TCP))
1339         {
1340           if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1341             {
1342               old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1343               new_port1 = udp1->src_port = s1->out2in.port;
1344               sum1 = tcp1->checksum;
1345               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1346                                      ip4_header_t,
1347                                      dst_address /* changed member */ );
1348               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1349                                      ip4_header_t /* cheat */ ,
1350                                      length /* changed member */ );
1351               mss_clamping (nm->mss_clamping, tcp1, &sum1);
1352               tcp1->checksum = ip_csum_fold (sum1);
1353             }
1354           vlib_increment_simple_counter (is_slow_path ?
1355                                            &nm->counters.slowpath.in2out.tcp :
1356                                            &nm->counters.fastpath.in2out.tcp,
1357                                          thread_index, cntr_sw_if_index1, 1);
1358         }
1359       else
1360         {
1361           if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1362             {
1363               udp1->src_port = s1->out2in.port;
1364               if (PREDICT_FALSE (udp1->checksum))
1365                 {
1366                   old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1367                   new_port1 = udp1->src_port;
1368                   sum1 = udp1->checksum;
1369                   sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t, dst_address  /* changed member */
1370                     );
1371                   sum1 =
1372                     ip_csum_update (sum1, old_port1, new_port1,
1373                                     ip4_header_t /* cheat */ ,
1374                                     length /* changed member */ );
1375                   udp1->checksum = ip_csum_fold (sum1);
1376                 }
1377             }
1378           vlib_increment_simple_counter (is_slow_path ?
1379                                            &nm->counters.slowpath.in2out.udp :
1380                                            &nm->counters.fastpath.in2out.udp,
1381                                          thread_index, cntr_sw_if_index1, 1);
1382         }
1383
1384       /* Accounting */
1385       nat44_ei_session_update_counters (
1386         s1, now, vlib_buffer_length_in_chain (vm, b1), thread_index);
1387       /* Per-user LRU list maintenance */
1388       nat44_ei_session_update_lru (nm, s1, thread_index);
1389     trace01:
1390
1391       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1392                          && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1393         {
1394           nat44_ei_in2out_trace_t *t =
1395             vlib_add_trace (vm, node, b1, sizeof (*t));
1396           t->sw_if_index = rx_sw_if_index1;
1397           t->next_index = next1;
1398           t->session_index = ~0;
1399           if (s1)
1400             t->session_index = s1 - nm->per_thread_data[thread_index].sessions;
1401         }
1402
1403       if (next1 == NAT44_EI_IN2OUT_NEXT_DROP)
1404         {
1405           vlib_increment_simple_counter (
1406             is_slow_path ? &nm->counters.slowpath.in2out.drops :
1407                            &nm->counters.fastpath.in2out.drops,
1408             thread_index, cntr_sw_if_index1, 1);
1409         }
1410
1411       n_left_from -= 2;
1412       next[0] = next0;
1413       next[1] = next1;
1414       next += 2;
1415     }
1416
1417   while (n_left_from > 0)
1418     {
1419       vlib_buffer_t *b0;
1420       u32 next0;
1421       u32 rx_sw_if_index0;
1422       u32 tx_sw_if_index0;
1423       u32 cntr_sw_if_index0;
1424       ip4_header_t *ip0;
1425       ip_csum_t sum0;
1426       u32 new_addr0, old_addr0;
1427       u16 old_port0, new_port0;
1428       udp_header_t *udp0;
1429       tcp_header_t *tcp0;
1430       icmp46_header_t *icmp0;
1431       u32 rx_fib_index0;
1432       u32 proto0;
1433       nat44_ei_session_t *s0 = 0;
1434       clib_bihash_kv_8_8_t kv0, value0;
1435       u32 iph_offset0 = 0;
1436
1437       b0 = *b;
1438       b++;
1439       next0 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
1440
1441       if (is_output_feature)
1442         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1443
1444       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1445                               iph_offset0);
1446
1447       udp0 = ip4_next_header (ip0);
1448       tcp0 = (tcp_header_t *) udp0;
1449       icmp0 = (icmp46_header_t *) udp0;
1450
1451       rx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1452       tx_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1453       cntr_sw_if_index0 =
1454         is_output_feature ? tx_sw_if_index0 : rx_sw_if_index0;
1455       rx_fib_index0 =
1456         vec_elt (nm->ip4_main->fib_index_by_sw_if_index, rx_sw_if_index0);
1457
1458       if (PREDICT_FALSE (ip0->ttl == 1))
1459         {
1460           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1461           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1462                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1463                                        0);
1464           next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1465           goto trace0;
1466         }
1467
1468       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1469
1470       /* Next configured feature, probably ip4-lookup */
1471       if (is_slow_path)
1472         {
1473           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1474             {
1475               if (nat_in2out_sm_unknown_proto (nm, b0, ip0, rx_fib_index0))
1476                 {
1477                   next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1478                   b0->error =
1479                     node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1480                 }
1481               vlib_increment_simple_counter (
1482                 is_slow_path ? &nm->counters.slowpath.in2out.other :
1483                                &nm->counters.fastpath.in2out.other,
1484                 thread_index, cntr_sw_if_index0, 1);
1485               goto trace0;
1486             }
1487
1488           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1489             {
1490               next0 = nat44_ei_icmp_in2out_slow_path (
1491                 nm, b0, ip0, icmp0, rx_sw_if_index0, rx_fib_index0, node,
1492                 next0, now, thread_index, &s0);
1493               vlib_increment_simple_counter (
1494                 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1495                                &nm->counters.fastpath.in2out.icmp,
1496                 thread_index, cntr_sw_if_index0, 1);
1497               goto trace0;
1498             }
1499         }
1500       else
1501         {
1502           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1503             {
1504               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1505               goto trace0;
1506             }
1507
1508           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1509             {
1510               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1511               goto trace0;
1512             }
1513         }
1514
1515       init_nat_k (&kv0, ip0->src_address,
1516                   vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1517                   proto0);
1518
1519       if (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
1520         {
1521           if (is_slow_path)
1522             {
1523               if (is_output_feature)
1524                 {
1525                   if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1526                         nm, ip0, proto0,
1527                         vnet_buffer (b0)->ip.reass.l4_src_port,
1528                         vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1529                         rx_sw_if_index0)))
1530                     goto trace0;
1531
1532                   /*
1533                    * Send DHCP packets to the ipv4 stack, or we won't
1534                    * be able to use dhcp client on the outside interface
1535                    */
1536                   if (PREDICT_FALSE
1537                       (proto0 == NAT_PROTOCOL_UDP
1538                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1539                            clib_host_to_net_u16
1540                            (UDP_DST_PORT_dhcp_to_server))
1541                        && ip0->dst_address.as_u32 == 0xffffffff))
1542                     goto trace0;
1543                 }
1544               else
1545                 {
1546                   if (PREDICT_FALSE (nat44_ei_not_translate (
1547                         nm, node, rx_sw_if_index0, ip0, proto0, rx_fib_index0,
1548                         thread_index)))
1549                     goto trace0;
1550                 }
1551
1552               next0 = slow_path (nm, b0, ip0, ip0->src_address,
1553                                  vnet_buffer (b0)->ip.reass.l4_src_port,
1554                                  rx_fib_index0, proto0, &s0, node, next0,
1555                                  thread_index, now);
1556
1557               if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
1558                 goto trace0;
1559
1560               if (PREDICT_FALSE (!s0))
1561                 goto trace0;
1562             }
1563           else
1564             {
1565               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1566               goto trace0;
1567             }
1568         }
1569       else
1570         s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1571                                 nat_value_get_session_index (&value0));
1572
1573       b0->flags |= VNET_BUFFER_F_IS_NATED;
1574
1575       old_addr0 = ip0->src_address.as_u32;
1576       ip0->src_address = s0->out2in.addr;
1577       new_addr0 = ip0->src_address.as_u32;
1578       if (!is_output_feature)
1579         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1580
1581       sum0 = ip0->checksum;
1582       sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1583                              ip4_header_t, src_address /* changed member */ );
1584       ip0->checksum = ip_csum_fold (sum0);
1585
1586       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1587         {
1588           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1589             {
1590               old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1591               new_port0 = udp0->src_port = s0->out2in.port;
1592               sum0 = tcp0->checksum;
1593               sum0 =
1594                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1595                                 dst_address /* changed member */ );
1596               sum0 =
1597                 ip_csum_update (sum0, old_port0, new_port0,
1598                                 ip4_header_t /* cheat */ ,
1599                                 length /* changed member */ );
1600               mss_clamping (nm->mss_clamping, tcp0, &sum0);
1601               tcp0->checksum = ip_csum_fold (sum0);
1602             }
1603           vlib_increment_simple_counter (is_slow_path ?
1604                                            &nm->counters.slowpath.in2out.tcp :
1605                                            &nm->counters.fastpath.in2out.tcp,
1606                                          thread_index, cntr_sw_if_index0, 1);
1607         }
1608       else
1609         {
1610           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1611             {
1612               udp0->src_port = s0->out2in.port;
1613               if (PREDICT_FALSE (udp0->checksum))
1614                 {
1615                   old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1616                   new_port0 = udp0->src_port;
1617                   sum0 = udp0->checksum;
1618                   sum0 =
1619                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1620                                     dst_address /* changed member */ );
1621                   sum0 =
1622                     ip_csum_update (sum0, old_port0, new_port0,
1623                                     ip4_header_t /* cheat */ ,
1624                                     length /* changed member */ );
1625                   udp0->checksum = ip_csum_fold (sum0);
1626                 }
1627             }
1628           vlib_increment_simple_counter (is_slow_path ?
1629                                            &nm->counters.slowpath.in2out.udp :
1630                                            &nm->counters.fastpath.in2out.udp,
1631                                          thread_index, cntr_sw_if_index0, 1);
1632         }
1633
1634       /* Accounting */
1635       nat44_ei_session_update_counters (
1636         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1637       /* Per-user LRU list maintenance */
1638       nat44_ei_session_update_lru (nm, s0, thread_index);
1639
1640     trace0:
1641       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1642                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1643         {
1644           nat44_ei_in2out_trace_t *t =
1645             vlib_add_trace (vm, node, b0, sizeof (*t));
1646           t->is_slow_path = is_slow_path;
1647           t->sw_if_index = rx_sw_if_index0;
1648           t->next_index = next0;
1649           t->session_index = ~0;
1650           if (s0)
1651             t->session_index = s0 - nm->per_thread_data[thread_index].sessions;
1652         }
1653
1654       if (next0 == NAT44_EI_IN2OUT_NEXT_DROP)
1655         {
1656           vlib_increment_simple_counter (
1657             is_slow_path ? &nm->counters.slowpath.in2out.drops :
1658                            &nm->counters.fastpath.in2out.drops,
1659             thread_index, cntr_sw_if_index0, 1);
1660         }
1661
1662       n_left_from--;
1663       next[0] = next0;
1664       next++;
1665     }
1666
1667   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1668                                frame->n_vectors);
1669   return frame->n_vectors;
1670 }
1671
1672 VLIB_NODE_FN (nat44_ei_in2out_node)
1673 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1674 {
1675   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */,
1676                                          0);
1677 }
1678
1679 VLIB_REGISTER_NODE (nat44_ei_in2out_node) = {
1680   .name = "nat44-ei-in2out",
1681   .vector_size = sizeof (u32),
1682   .format_trace = format_nat44_ei_in2out_trace,
1683   .type = VLIB_NODE_TYPE_INTERNAL,
1684
1685   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1686   .error_strings = nat44_ei_in2out_error_strings,
1687
1688   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1689
1690   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1691
1692   /* edit / add dispositions here */
1693   .next_nodes = {
1694     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1695     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1696     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
1697     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1698     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
1699   },
1700 };
1701
1702 VLIB_NODE_FN (nat44_ei_in2out_output_node)
1703 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1704 {
1705   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */,
1706                                          1);
1707 }
1708
1709 VLIB_REGISTER_NODE (nat44_ei_in2out_output_node) = {
1710   .name = "nat44-ei-in2out-output",
1711   .vector_size = sizeof (u32),
1712   .format_trace = format_nat44_ei_in2out_trace,
1713   .type = VLIB_NODE_TYPE_INTERNAL,
1714
1715   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1716   .error_strings = nat44_ei_in2out_error_strings,
1717
1718   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1719
1720   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1721
1722   /* edit / add dispositions here */
1723   .next_nodes = {
1724     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1725     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "interface-output",
1726     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-output-slowpath",
1727     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1728     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-interface-output",
1729   },
1730 };
1731
1732 VLIB_NODE_FN (nat44_ei_in2out_slowpath_node)
1733 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1734 {
1735   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */,
1736                                          0);
1737 }
1738
1739 VLIB_REGISTER_NODE (nat44_ei_in2out_slowpath_node) = {
1740   .name = "nat44-ei-in2out-slowpath",
1741   .vector_size = sizeof (u32),
1742   .format_trace = format_nat44_ei_in2out_trace,
1743   .type = VLIB_NODE_TYPE_INTERNAL,
1744
1745   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1746   .error_strings = nat44_ei_in2out_error_strings,
1747
1748   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1749
1750   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1751
1752   /* edit / add dispositions here */
1753   .next_nodes = {
1754     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1755     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1756     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
1757     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1758     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
1759   },
1760 };
1761
1762 VLIB_NODE_FN (nat44_ei_in2out_output_slowpath_node)
1763 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1764 {
1765   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */,
1766                                          1);
1767 }
1768
1769 VLIB_REGISTER_NODE (nat44_ei_in2out_output_slowpath_node) = {
1770   .name = "nat44-ei-in2out-output-slowpath",
1771   .vector_size = sizeof (u32),
1772   .format_trace = format_nat44_ei_in2out_trace,
1773   .type = VLIB_NODE_TYPE_INTERNAL,
1774
1775   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1776   .error_strings = nat44_ei_in2out_error_strings,
1777
1778   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1779
1780   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1781
1782   /* edit / add dispositions here */
1783   .next_nodes = {
1784     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1785     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "interface-output",
1786     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-output-slowpath",
1787     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1788     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-interface-output",
1789   },
1790 };
1791
1792 VLIB_NODE_FN (nat44_ei_in2out_fast_node)
1793 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1794 {
1795   u32 n_left_from, *from, *to_next;
1796   u32 thread_index = vm->thread_index;
1797   nat44_ei_in2out_next_t next_index;
1798   nat44_ei_main_t *nm = &nat44_ei_main;
1799   int is_hairpinning = 0;
1800
1801   from = vlib_frame_vector_args (frame);
1802   n_left_from = frame->n_vectors;
1803   next_index = node->cached_next_index;
1804
1805   while (n_left_from > 0)
1806     {
1807       u32 n_left_to_next;
1808
1809       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1810
1811       while (n_left_from > 0 && n_left_to_next > 0)
1812         {
1813           u32 bi0;
1814           vlib_buffer_t *b0;
1815           u32 next0;
1816           u32 sw_if_index0;
1817           ip4_header_t *ip0;
1818           ip_csum_t sum0;
1819           u32 new_addr0, old_addr0;
1820           u16 old_port0, new_port0;
1821           udp_header_t *udp0;
1822           tcp_header_t *tcp0;
1823           icmp46_header_t *icmp0;
1824           u32 proto0;
1825           u32 rx_fib_index0;
1826           ip4_address_t sm0_addr;
1827           u16 sm0_port;
1828           u32 sm0_fib_index;
1829           u32 required_thread_index = thread_index;
1830
1831           /* speculatively enqueue b0 to the current next frame */
1832           bi0 = from[0];
1833           to_next[0] = bi0;
1834           from += 1;
1835           to_next += 1;
1836           n_left_from -= 1;
1837           n_left_to_next -= 1;
1838
1839           b0 = vlib_get_buffer (vm, bi0);
1840           next0 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
1841
1842           ip0 = vlib_buffer_get_current (b0);
1843           udp0 = ip4_next_header (ip0);
1844           tcp0 = (tcp_header_t *) udp0;
1845           icmp0 = (icmp46_header_t *) udp0;
1846
1847           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1848           rx_fib_index0 =
1849             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1850
1851           if (PREDICT_FALSE (ip0->ttl == 1))
1852             {
1853               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1854               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1855                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1856                                            0);
1857               next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1858               goto trace0;
1859             }
1860
1861           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1862
1863           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1864             goto trace0;
1865
1866           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1867             {
1868               next0 = nat44_ei_icmp_in2out (b0, ip0, icmp0, sw_if_index0,
1869                                             rx_fib_index0, node, next0, ~0, 0);
1870               goto trace0;
1871             }
1872
1873           if (nat44_ei_static_mapping_match (
1874                 ip0->src_address, udp0->src_port, rx_fib_index0, proto0,
1875                 &sm0_addr, &sm0_port, &sm0_fib_index, 0, 0, 0))
1876             {
1877               b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_NO_TRANSLATION];
1878               next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1879               goto trace0;
1880             }
1881
1882           new_addr0 = sm0_addr.as_u32;
1883           new_port0 = sm0_port;
1884           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0_fib_index;
1885           old_addr0 = ip0->src_address.as_u32;
1886           ip0->src_address.as_u32 = new_addr0;
1887
1888           sum0 = ip0->checksum;
1889           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1890                                  ip4_header_t,
1891                                  src_address /* changed member */ );
1892           ip0->checksum = ip_csum_fold (sum0);
1893
1894           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1895             {
1896               old_port0 = udp0->src_port;
1897               udp0->src_port = new_port0;
1898
1899               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1900                 {
1901                   sum0 = tcp0->checksum;
1902                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1903                                          ip4_header_t,
1904                                          dst_address /* changed member */ );
1905                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1906                                          ip4_header_t /* cheat */ ,
1907                                          length /* changed member */ );
1908                   mss_clamping (nm->mss_clamping, tcp0, &sum0);
1909                   tcp0->checksum = ip_csum_fold (sum0);
1910                 }
1911               else if (udp0->checksum)
1912                 {
1913                   sum0 = udp0->checksum;
1914                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1915                                          ip4_header_t,
1916                                          dst_address /* changed member */ );
1917                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1918                                          ip4_header_t /* cheat */ ,
1919                                          length /* changed member */ );
1920                   udp0->checksum = ip_csum_fold (sum0);
1921                 }
1922             }
1923           else
1924             {
1925               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1926                 {
1927                   sum0 = tcp0->checksum;
1928                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1929                                          ip4_header_t,
1930                                          dst_address /* changed member */ );
1931                   mss_clamping (nm->mss_clamping, tcp0, &sum0);
1932                   tcp0->checksum = ip_csum_fold (sum0);
1933                 }
1934               else if (udp0->checksum)
1935                 {
1936                   sum0 = udp0->checksum;
1937                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1938                                          ip4_header_t,
1939                                          dst_address /* changed member */ );
1940                   udp0->checksum = ip_csum_fold (sum0);
1941                 }
1942             }
1943
1944           /* Hairpinning */
1945           is_hairpinning = nat44_ei_hairpinning (
1946             vm, node, nm, thread_index, b0, ip0, udp0, tcp0, proto0,
1947             0 /* do_trace */, &required_thread_index);
1948
1949           if (thread_index != required_thread_index)
1950             {
1951               vnet_buffer (b0)->snat.required_thread_index =
1952                 required_thread_index;
1953               next0 = NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF;
1954             }
1955
1956         trace0:
1957           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1958                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1959             {
1960               nat44_ei_in2out_trace_t *t =
1961                 vlib_add_trace (vm, node, b0, sizeof (*t));
1962               t->sw_if_index = sw_if_index0;
1963               t->next_index = next0;
1964               t->is_hairpinning = is_hairpinning;
1965             }
1966
1967           if (next0 != NAT44_EI_IN2OUT_NEXT_DROP)
1968             {
1969
1970               vlib_increment_simple_counter (
1971                 &nm->counters.fastpath.in2out.other, sw_if_index0,
1972                 vm->thread_index, 1);
1973             }
1974
1975           /* verify speculative enqueue, maybe switch current next frame */
1976           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1977                                            to_next, n_left_to_next,
1978                                            bi0, next0);
1979         }
1980
1981       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1982     }
1983
1984   return frame->n_vectors;
1985 }
1986
1987 VLIB_REGISTER_NODE (nat44_ei_in2out_fast_node) = {
1988   .name = "nat44-ei-in2out-fast",
1989   .vector_size = sizeof (u32),
1990   .format_trace = format_nat44_ei_in2out_fast_trace,
1991   .type = VLIB_NODE_TYPE_INTERNAL,
1992
1993   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1994   .error_strings = nat44_ei_in2out_error_strings,
1995
1996   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1997
1998   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1999
2000   /* edit / add dispositions here */
2001   .next_nodes = {
2002     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
2003     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2004     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
2005     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2006     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2007   },
2008 };
2009
2010 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_handoff_ip4_lookup_node)
2011 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2012 {
2013   return nat44_ei_hairpinning_handoff_fn_inline (
2014     vm, node, frame,
2015     nat44_ei_main.in2out_hairpinning_finish_ip4_lookup_node_fq_index);
2016 }
2017
2018 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_handoff_ip4_lookup_node) = {
2019   .name = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2020   .vector_size = sizeof (u32),
2021   .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2022   .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2023   .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2024
2025   .n_next_nodes = 1,
2026
2027   .next_nodes = {
2028     [0] = "error-drop",
2029   },
2030 };
2031
2032 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_handoff_interface_output_node)
2033 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2034 {
2035   return nat44_ei_hairpinning_handoff_fn_inline (
2036     vm, node, frame,
2037     nat44_ei_main.in2out_hairpinning_finish_interface_output_node_fq_index);
2038 }
2039
2040 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_handoff_interface_output_node) = {
2041   .name = "nat44-ei-in2out-hairpinning-handoff-interface-output",
2042   .vector_size = sizeof (u32),
2043   .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2044   .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2045   .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2046
2047   .n_next_nodes = 1,
2048
2049   .next_nodes = {
2050     [0] = "error-drop",
2051   },
2052 };
2053
2054 static_always_inline int
2055 nat44_ei_in2out_hairpinning_finish_inline (vlib_main_t *vm,
2056                                            vlib_node_runtime_t *node,
2057                                            vlib_frame_t *frame)
2058 {
2059   u32 n_left_from, *from, *to_next;
2060   u32 thread_index = vm->thread_index;
2061   nat44_ei_in2out_next_t next_index;
2062   nat44_ei_main_t *nm = &nat44_ei_main;
2063   int is_hairpinning = 0;
2064
2065   from = vlib_frame_vector_args (frame);
2066   n_left_from = frame->n_vectors;
2067   next_index = node->cached_next_index;
2068
2069   while (n_left_from > 0)
2070     {
2071       u32 n_left_to_next;
2072
2073       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2074
2075       while (n_left_from > 0 && n_left_to_next > 0)
2076         {
2077           u32 bi0;
2078           vlib_buffer_t *b0;
2079           u32 next0;
2080           u32 sw_if_index0;
2081           ip4_header_t *ip0;
2082           udp_header_t *udp0;
2083           tcp_header_t *tcp0;
2084           icmp46_header_t *icmp0;
2085           u32 proto0;
2086           u32 required_thread_index = thread_index;
2087
2088           /* speculatively enqueue b0 to the current next frame */
2089           bi0 = from[0];
2090           to_next[0] = bi0;
2091           from += 1;
2092           to_next += 1;
2093           n_left_from -= 1;
2094           n_left_to_next -= 1;
2095
2096           b0 = vlib_get_buffer (vm, bi0);
2097           next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP;
2098
2099           ip0 = vlib_buffer_get_current (b0);
2100           udp0 = ip4_next_header (ip0);
2101           tcp0 = (tcp_header_t *) udp0;
2102           icmp0 = (icmp46_header_t *) udp0;
2103
2104           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2105           proto0 = ip_proto_to_nat_proto (ip0->protocol);
2106
2107           switch (proto0)
2108             {
2109             case NAT_PROTOCOL_TCP:
2110               // fallthrough
2111             case NAT_PROTOCOL_UDP:
2112               is_hairpinning = nat44_ei_hairpinning (
2113                 vm, node, nm, thread_index, b0, ip0, udp0, tcp0, proto0,
2114                 0 /* do_trace */, &required_thread_index);
2115               break;
2116             case NAT_PROTOCOL_ICMP:
2117               is_hairpinning = (0 == nat44_ei_icmp_hairpinning (
2118                                        nm, b0, thread_index, ip0, icmp0,
2119                                        &required_thread_index));
2120               break;
2121             case NAT_PROTOCOL_OTHER:
2122               // this should never happen
2123               next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2124               break;
2125             }
2126
2127           if (thread_index != required_thread_index)
2128             {
2129               // but we already did a handoff ...
2130               next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2131             }
2132
2133           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
2134                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
2135             {
2136               nat44_ei_in2out_trace_t *t =
2137                 vlib_add_trace (vm, node, b0, sizeof (*t));
2138               t->sw_if_index = sw_if_index0;
2139               t->next_index = next0;
2140               t->is_hairpinning = is_hairpinning;
2141             }
2142
2143           if (next0 != NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP)
2144             {
2145               vlib_increment_simple_counter (
2146                 &nm->counters.fastpath.in2out.other, sw_if_index0,
2147                 vm->thread_index, 1);
2148             }
2149
2150           /* verify speculative enqueue, maybe switch current next frame */
2151           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2152                                            n_left_to_next, bi0, next0);
2153         }
2154
2155       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2156     }
2157
2158   return frame->n_vectors;
2159 }
2160
2161 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_finish_ip4_lookup_node)
2162 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2163 {
2164   return nat44_ei_in2out_hairpinning_finish_inline (vm, node, frame);
2165 }
2166
2167 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_finish_ip4_lookup_node) = {
2168   .name = "nat44-ei-in2out-hairpinning-finish-ip4-lookup",
2169   .vector_size = sizeof (u32),
2170   .format_trace = format_nat44_ei_in2out_fast_trace,
2171   .type = VLIB_NODE_TYPE_INTERNAL,
2172
2173   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2174   .error_strings = nat44_ei_in2out_error_strings,
2175
2176   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2177
2178   .n_next_nodes = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2179
2180   /* edit / add dispositions here */
2181   .next_nodes = {
2182     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2183     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "ip4-lookup",
2184   },
2185 };
2186
2187 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_finish_interface_output_node)
2188 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2189 {
2190   return nat44_ei_in2out_hairpinning_finish_inline (vm, node, frame);
2191 }
2192
2193 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_finish_interface_output_node) = {
2194   .name = "nat44-ei-in2out-hairpinning-finish-interface-output",
2195   .vector_size = sizeof (u32),
2196   .format_trace = format_nat44_ei_in2out_fast_trace,
2197   .type = VLIB_NODE_TYPE_INTERNAL,
2198
2199   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2200   .error_strings = nat44_ei_in2out_error_strings,
2201
2202   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2203
2204   .n_next_nodes = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2205
2206   /* edit / add dispositions here */
2207   .next_nodes = {
2208     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2209     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "interface-output",
2210   },
2211 };
2212
2213 /*
2214  * fd.io coding-style-patch-verification: ON
2215  *
2216  * Local Variables:
2217  * eval: (c-set-style "gnu")
2218  * End:
2219  */