src/plugins/nat/nat44-ei/nat44_ei_in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /**
  16  * @file
  17  * @brief NAT44 EI inside to outside network translation
  18  */
  19
  20 #include <vlib/vlib.h>
  21
  22 #include <vnet/vnet.h>
  23 #include <vnet/ip/ip.h>
  24 #include <vnet/ethernet/ethernet.h>
  25 #include <vnet/udp/udp_local.h>
  26 #include <vnet/fib/ip4_fib.h>
  27
  28 #include <vppinfra/hash.h>
  29 #include <vppinfra/error.h>
  30
  31 #include <nat/lib/log.h>
  32 #include <nat/lib/nat_syslog.h>
  33 #include <nat/lib/ipfix_logging.h>
  34 #include <nat/lib/nat_inlines.h>
  35 #include <nat/nat44-ei/nat44_ei_inlines.h>
  36 #include <nat/nat44-ei/nat44_ei.h>
  37 #include <nat/nat44-ei/nat44_ei_hairpinning.h>
  38
  39 typedef struct
  40 {
  41   u32 sw_if_index;
  42   u32 next_index;
  43   u32 session_index;
  44   u32 is_slow_path;
  45   u32 is_hairpinning;
  46 } nat44_ei_in2out_trace_t;
  47
  48 /* packet trace format function */
  49 static u8 *
  50 format_nat44_ei_in2out_trace (u8 *s, va_list *args)
  51 {
  52   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  53   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  54   nat44_ei_in2out_trace_t *t = va_arg (*args, nat44_ei_in2out_trace_t *);
  55   char *tag;
  56
  57   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  58
  59   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  60               t->sw_if_index, t->next_index, t->session_index);
  61   if (t->is_hairpinning)
  62     {
  63       s = format (s, ", with-hairpinning");
  64     }
  65
  66   return s;
  67 }
  68
  69 static u8 *
  70 format_nat44_ei_in2out_fast_trace (u8 *s, va_list *args)
  71 {
  72   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  73   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  74   nat44_ei_in2out_trace_t *t = va_arg (*args, nat44_ei_in2out_trace_t *);
  75
  76   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  77               t->sw_if_index, t->next_index);
  78
  79   return s;
  80 }
  81
  82 #define foreach_nat44_ei_in2out_error                                         \
  83   _ (UNSUPPORTED_PROTOCOL, "unsupported protocol")                            \
  84   _ (OUT_OF_PORTS, "out of ports")                                            \
  85   _ (BAD_OUTSIDE_FIB, "outside VRF ID not found")                             \
  86   _ (BAD_ICMP_TYPE, "unsupported ICMP type")                                  \
  87   _ (NO_TRANSLATION, "no translation")                                        \
  88   _ (MAX_SESSIONS_EXCEEDED, "maximum sessions exceeded")                      \
  89   _ (CANNOT_CREATE_USER, "cannot create NAT user")
  90
  91 typedef enum
  92 {
  93 #define _(sym, str) NAT44_EI_IN2OUT_ERROR_##sym,
  94   foreach_nat44_ei_in2out_error
  95 #undef _
  96     NAT44_EI_IN2OUT_N_ERROR,
  97 } nat44_ei_in2out_error_t;
  98
  99 static char *nat44_ei_in2out_error_strings[] = {
 100 #define _(sym,string) string,
 101   foreach_nat44_ei_in2out_error
 102 #undef _
 103 };
 104
 105 typedef enum
 106 {
 107   NAT44_EI_IN2OUT_NEXT_LOOKUP,
 108   NAT44_EI_IN2OUT_NEXT_DROP,
 109   NAT44_EI_IN2OUT_NEXT_ICMP_ERROR,
 110   NAT44_EI_IN2OUT_NEXT_SLOW_PATH,
 111   NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF,
 112   NAT44_EI_IN2OUT_N_NEXT,
 113 } nat44_ei_in2out_next_t;
 114
 115 typedef enum
 116 {
 117   NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP,
 118   NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP,
 119   NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
 120 } nat44_ei_in2out_hairpinnig_finish_next_t;
 121
 122 static inline int
 123 nat44_ei_not_translate_fast (vlib_node_runtime_t *node, u32 sw_if_index0,
 124                              ip4_header_t *ip0, u32 proto0, u32 rx_fib_index0)
 125 {
 126   nat44_ei_main_t *nm = &nat44_ei_main;
 127
 128   if (nm->out2in_dpo)
 129     return 0;
 130
 131   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 132   nat44_ei_outside_fib_t *outside_fib;
 133   fib_prefix_t pfx = {
 134     .fp_proto = FIB_PROTOCOL_IP4,
 135     .fp_len = 32,
 136     .fp_addr = {
 137                 .ip4.as_u32 = ip0->dst_address.as_u32,
 138                 }
 139     ,
 140   };
 141
 142   /* Don't NAT packet aimed at the intfc address */
 143   if (PREDICT_FALSE (nat44_ei_is_interface_addr (
 144         nm->ip4_main, node, sw_if_index0, ip0->dst_address.as_u32)))
 145     return 1;
 146
 147   fei = fib_table_lookup (rx_fib_index0, &pfx);
 148   if (FIB_NODE_INDEX_INVALID != fei)
 149     {
 150       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 151       if (sw_if_index == ~0)
 152         {
 153           vec_foreach (outside_fib, nm->outside_fibs)
 154             {
 155               fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 156               if (FIB_NODE_INDEX_INVALID != fei)
 157                 {
 158                   sw_if_index = fib_entry_get_resolving_interface (fei);
 159                   if (sw_if_index != ~0)
 160                     break;
 161                 }
 162             }
 163         }
 164       if (sw_if_index == ~0)
 165         return 1;
 166
 167       nat44_ei_interface_t *i;
 168       pool_foreach (i, nm->interfaces)
 169         {
 170           /* NAT packet aimed at outside interface */
 171           if ((nat44_ei_interface_is_outside (i)) &&
 172               (sw_if_index == i->sw_if_index))
 173             return 0;
 174         }
 175     }
 176
 177   return 1;
 178 }
 179
 180 static inline int
 181 nat44_ei_not_translate (nat44_ei_main_t *nm, vlib_node_runtime_t *node,
 182                         u32 sw_if_index0, ip4_header_t *ip0, u32 proto0,
 183                         u32 rx_fib_index0, u32 thread_index)
 184 {
 185   udp_header_t *udp0 = ip4_next_header (ip0);
 186   clib_bihash_kv_8_8_t kv0, value0;
 187
 188   init_nat_k (&kv0, ip0->dst_address, udp0->dst_port, nm->outside_fib_index,
 189               proto0);
 190
 191   /* NAT packet aimed at external address if */
 192   /* has active sessions */
 193   if (clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
 194     {
 195       /* or is static mappings */
 196       ip4_address_t placeholder_addr;
 197       u16 placeholder_port;
 198       u32 placeholder_fib_index;
 199       if (!nat44_ei_static_mapping_match (ip0->dst_address, udp0->dst_port,
 200                                           nm->outside_fib_index, proto0,
 201                                           &placeholder_addr, &placeholder_port,
 202                                           &placeholder_fib_index, 1, 0, 0))
 203         return 0;
 204     }
 205   else
 206     return 0;
 207
 208   if (nm->forwarding_enabled)
 209     return 1;
 210
 211   return nat44_ei_not_translate_fast (node, sw_if_index0, ip0, proto0,
 212                                       rx_fib_index0);
 213 }
 214
 215 static inline int
 216 nat44_ei_not_translate_output_feature (nat44_ei_main_t *nm, ip4_header_t *ip0,
 217                                        u32 proto0, u16 src_port, u16 dst_port,
 218                                        u32 thread_index, u32 sw_if_index)
 219 {
 220   clib_bihash_kv_8_8_t kv0, value0;
 221   nat44_ei_interface_t *i;
 222
 223   /* src NAT check */
 224   init_nat_k (&kv0, ip0->src_address, src_port,
 225               ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
 226
 227   if (!clib_bihash_search_8_8 (&nm->out2in, &kv0, &value0))
 228     return 1;
 229
 230   /* dst NAT check */
 231   init_nat_k (&kv0, ip0->dst_address, dst_port,
 232               ip4_fib_table_get_index_for_sw_if_index (sw_if_index), proto0);
 233   if (!clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
 234     {
 235       /* hairpinning */
 236       pool_foreach (i, nm->output_feature_interfaces)
 237         {
 238           if ((nat44_ei_interface_is_inside (i)) &&
 239               (sw_if_index == i->sw_if_index))
 240             return 0;
 241         }
 242       return 1;
 243     }
 244
 245   return 0;
 246 }
 247
 248 #ifndef CLIB_MARCH_VARIANT
 249 int
 250 nat44_i2o_is_idle_session_cb (clib_bihash_kv_8_8_t * kv, void *arg)
 251 {
 252   nat44_ei_main_t *nm = &nat44_ei_main;
 253   nat44_ei_is_idle_session_ctx_t *ctx = arg;
 254   nat44_ei_session_t *s;
 255   u64 sess_timeout_time;
 256   nat44_ei_main_per_thread_data_t *tnm =
 257     vec_elt_at_index (nm->per_thread_data, ctx->thread_index);
 258   clib_bihash_kv_8_8_t s_kv;
 259
 260   s = pool_elt_at_index (tnm->sessions, kv->value);
 261   sess_timeout_time = s->last_heard + (f64) nat_session_get_timeout (
 262                                         &nm->timeouts, s->nat_proto, s->state);
 263   if (ctx->now >= sess_timeout_time)
 264     {
 265       init_nat_o2i_k (&s_kv, s);
 266       if (clib_bihash_add_del_8_8 (&nm->out2in, &s_kv, 0))
 267         nat_elog_warn (nm, "out2in key del failed");
 268
 269       nat_ipfix_logging_nat44_ses_delete (ctx->thread_index,
 270                                           s->in2out.addr.as_u32,
 271                                           s->out2in.addr.as_u32,
 272                                           s->nat_proto,
 273                                           s->in2out.port,
 274                                           s->out2in.port,
 275                                           s->in2out.fib_index);
 276
 277       nat_syslog_nat44_apmdel (s->user_index, s->in2out.fib_index,
 278                                &s->in2out.addr, s->in2out.port,
 279                                &s->out2in.addr, s->out2in.port, s->nat_proto);
 280
 281       nat_ha_sdel (&s->out2in.addr, s->out2in.port, &s->ext_host_addr,
 282                    s->ext_host_port, s->nat_proto, s->out2in.fib_index,
 283                    ctx->thread_index);
 284
 285       if (!nat44_ei_is_session_static (s))
 286         nat44_ei_free_outside_address_and_port (
 287           nm->addresses, ctx->thread_index, &s->out2in.addr, s->out2in.port,
 288           s->nat_proto);
 289
 290       nat44_ei_delete_session (nm, s, ctx->thread_index);
 291       return 1;
 292     }
 293
 294   return 0;
 295 }
 296 #endif
 297
 298 static u32
 299 slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0, ip4_header_t *ip0,
 300            ip4_address_t i2o_addr, u16 i2o_port, u32 rx_fib_index0,
 301            nat_protocol_t nat_proto, nat44_ei_session_t **sessionp,
 302            vlib_node_runtime_t *node, u32 next0, u32 thread_index, f64 now)
 303 {
 304   nat44_ei_user_t *u;
 305   nat44_ei_session_t *s = 0;
 306   clib_bihash_kv_8_8_t kv0;
 307   u8 is_sm = 0;
 308   nat44_ei_outside_fib_t *outside_fib;
 309   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 310   u8 identity_nat;
 311   fib_prefix_t pfx = {
 312     .fp_proto = FIB_PROTOCOL_IP4,
 313     .fp_len = 32,
 314     .fp_addr = {
 315                 .ip4.as_u32 = ip0->dst_address.as_u32,
 316                 },
 317   };
 318   nat44_ei_is_idle_session_ctx_t ctx0;
 319   ip4_address_t sm_addr;
 320   u16 sm_port;
 321   u32 sm_fib_index;
 322
 323   if (PREDICT_FALSE (nat44_ei_maximum_sessions_exceeded (nm, thread_index)))
 324     {
 325       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 326       nat_ipfix_logging_max_sessions (thread_index,
 327                                       nm->max_translations_per_thread);
 328       nat_elog_notice (nm, "maximum sessions exceeded");
 329       return NAT44_EI_IN2OUT_NEXT_DROP;
 330     }
 331
 332   /* First try to match static mapping by local address and port */
 333   if (nat44_ei_static_mapping_match (i2o_addr, i2o_port, rx_fib_index0,
 334                                      nat_proto, &sm_addr, &sm_port,
 335                                      &sm_fib_index, 0, 0, &identity_nat))
 336     {
 337       /* Try to create dynamic translation */
 338       if (nm->alloc_addr_and_port (
 339             nm->addresses, rx_fib_index0, thread_index, nat_proto, &sm_addr,
 340             &sm_port, nm->port_per_thread,
 341             nm->per_thread_data[thread_index].snat_thread_index))
 342         {
 343           b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_OUT_OF_PORTS];
 344           return NAT44_EI_IN2OUT_NEXT_DROP;
 345         }
 346     }
 347   else
 348     {
 349       if (PREDICT_FALSE (identity_nat))
 350         {
 351           *sessionp = s;
 352           return next0;
 353         }
 354
 355       is_sm = 1;
 356     }
 357
 358   u = nat44_ei_user_get_or_create (nm, &ip0->src_address, rx_fib_index0,
 359                                    thread_index);
 360   if (!u)
 361     {
 362       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_CANNOT_CREATE_USER];
 363       return NAT44_EI_IN2OUT_NEXT_DROP;
 364     }
 365
 366   s = nat44_ei_session_alloc_or_recycle (nm, u, thread_index, now);
 367   if (!s)
 368     {
 369       nat44_ei_delete_user_with_no_session (nm, u, thread_index);
 370       nat_elog_warn (nm, "create NAT session failed");
 371       return NAT44_EI_IN2OUT_NEXT_DROP;
 372     }
 373
 374   if (is_sm)
 375     s->flags |= NAT44_EI_SESSION_FLAG_STATIC_MAPPING;
 376   nat44_ei_user_session_increment (nm, u, is_sm);
 377   s->in2out.addr = i2o_addr;
 378   s->in2out.port = i2o_port;
 379   s->in2out.fib_index = rx_fib_index0;
 380   s->nat_proto = nat_proto;
 381   s->out2in.addr = sm_addr;
 382   s->out2in.port = sm_port;
 383   s->out2in.fib_index = nm->outside_fib_index;
 384   switch (vec_len (nm->outside_fibs))
 385     {
 386     case 0:
 387       s->out2in.fib_index = nm->outside_fib_index;
 388       break;
 389     case 1:
 390       s->out2in.fib_index = nm->outside_fibs[0].fib_index;
 391       break;
 392     default:
 393       vec_foreach (outside_fib, nm->outside_fibs)
 394         {
 395           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 396           if (FIB_NODE_INDEX_INVALID != fei)
 397             {
 398               if (fib_entry_get_resolving_interface (fei) != ~0)
 399                 {
 400                   s->out2in.fib_index = outside_fib->fib_index;
 401                   break;
 402                 }
 403             }
 404         }
 405       break;
 406     }
 407   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 408   s->ext_host_port = vnet_buffer (b0)->ip.reass.l4_dst_port;
 409   *sessionp = s;
 410
 411   /* Add to translation hashes */
 412   ctx0.now = now;
 413   ctx0.thread_index = thread_index;
 414   init_nat_i2o_kv (&kv0, s, thread_index,
 415                    s - nm->per_thread_data[thread_index].sessions);
 416   if (clib_bihash_add_or_overwrite_stale_8_8 (
 417         &nm->in2out, &kv0, nat44_i2o_is_idle_session_cb, &ctx0))
 418     nat_elog_notice (nm, "in2out key add failed");
 419
 420   init_nat_o2i_kv (&kv0, s, thread_index,
 421                    s - nm->per_thread_data[thread_index].sessions);
 422   if (clib_bihash_add_or_overwrite_stale_8_8 (
 423         &nm->out2in, &kv0, nat44_o2i_is_idle_session_cb, &ctx0))
 424     nat_elog_notice (nm, "out2in key add failed");
 425
 426   /* log NAT event */
 427   nat_ipfix_logging_nat44_ses_create (
 428     thread_index, s->in2out.addr.as_u32, s->out2in.addr.as_u32, s->nat_proto,
 429     s->in2out.port, s->out2in.port, s->in2out.fib_index);
 430
 431   nat_syslog_nat44_apmadd (s->user_index, s->in2out.fib_index, &s->in2out.addr,
 432                            s->in2out.port, &s->out2in.addr, s->out2in.port,
 433                            s->nat_proto);
 434
 435   nat_ha_sadd (&s->in2out.addr, s->in2out.port, &s->out2in.addr,
 436                s->out2in.port, &s->ext_host_addr, s->ext_host_port,
 437                &s->ext_host_nat_addr, s->ext_host_nat_port, s->nat_proto,
 438                s->in2out.fib_index, s->flags, thread_index, 0);
 439
 440   return next0;
 441 }
 442
 443 #ifndef CLIB_MARCH_VARIANT
 444 static_always_inline nat44_ei_in2out_error_t
 445 icmp_get_key (vlib_buffer_t *b, ip4_header_t *ip0, ip4_address_t *addr,
 446               u16 *port, nat_protocol_t *nat_proto)
 447 {
 448   icmp46_header_t *icmp0;
 449   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 450   ip4_header_t *inner_ip0 = 0;
 451   void *l4_header = 0;
 452   icmp46_header_t *inner_icmp0;
 453
 454   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 455   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 456
 457   if (!icmp_type_is_error_message
 458       (vnet_buffer (b)->ip.reass.icmp_type_or_tcp_flags))
 459     {
 460       *nat_proto = NAT_PROTOCOL_ICMP;
 461       *addr = ip0->src_address;
 462       *port = vnet_buffer (b)->ip.reass.l4_src_port;
 463     }
 464   else
 465     {
 466       inner_ip0 = (ip4_header_t *) (echo0 + 1);
 467       l4_header = ip4_next_header (inner_ip0);
 468       *nat_proto = ip_proto_to_nat_proto (inner_ip0->protocol);
 469       *addr = inner_ip0->dst_address;
 470       switch (*nat_proto)
 471         {
 472         case NAT_PROTOCOL_ICMP:
 473           inner_icmp0 = (icmp46_header_t *) l4_header;
 474           inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 475           *port = inner_echo0->identifier;
 476           break;
 477         case NAT_PROTOCOL_UDP:
 478         case NAT_PROTOCOL_TCP:
 479           *port = ((tcp_udp_header_t *) l4_header)->dst_port;
 480           break;
 481         default:
 482           return NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 483         }
 484     }
 485   return -1;                    /* success */
 486 }
 487
 488 /**
 489  * Get address and port values to be used for ICMP packet translation
 490  * and create session if needed
 491  *
 492  * @param[in,out] nm             NAT main
 493  * @param[in,out] node           NAT node runtime
 494  * @param[in] thread_index       thread index
 495  * @param[in,out] b0             buffer containing packet to be translated
 496  * @param[in,out] ip0            ip header
 497  * @param[out] p_proto           protocol used for matching
 498  * @param[out] p_value           address and port after NAT translation
 499  * @param[out] p_dont_translate  if packet should not be translated
 500  * @param d                      optional parameter
 501  * @param e                      optional parameter
 502  */
 503 u32
 504 nat44_ei_icmp_match_in2out_slow (vlib_node_runtime_t *node, u32 thread_index,
 505                                  vlib_buffer_t *b0, ip4_header_t *ip0,
 506                                  ip4_address_t *addr, u16 *port,
 507                                  u32 *fib_index, nat_protocol_t *proto,
 508                                  nat44_ei_session_t **p_s0, u8 *dont_translate)
 509 {
 510   nat44_ei_main_t *nm = &nat44_ei_main;
 511   nat44_ei_main_per_thread_data_t *tnm = &nm->per_thread_data[thread_index];
 512   u32 sw_if_index0;
 513   nat44_ei_session_t *s0 = 0;
 514   clib_bihash_kv_8_8_t kv0, value0;
 515   u32 next0 = ~0;
 516   int err;
 517   vlib_main_t *vm = vlib_get_main ();
 518   *dont_translate = 0;
 519
 520   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 521   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 522
 523   err = icmp_get_key (b0, ip0, addr, port, proto);
 524   if (err != -1)
 525     {
 526       b0->error = node->errors[err];
 527       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 528       goto out;
 529     }
 530
 531   init_nat_k (&kv0, *addr, *port, *fib_index, *proto);
 532   if (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
 533     {
 534       if (vnet_buffer (b0)->sw_if_index[VLIB_TX] != ~0)
 535         {
 536           if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
 537                 nm, ip0, *proto, *port, *port, thread_index, sw_if_index0)))
 538             {
 539               *dont_translate = 1;
 540               goto out;
 541             }
 542         }
 543       else
 544         {
 545           if (PREDICT_FALSE (nat44_ei_not_translate (
 546                 nm, node, sw_if_index0, ip0, NAT_PROTOCOL_ICMP, *fib_index,
 547                 thread_index)))
 548             {
 549               *dont_translate = 1;
 550               goto out;
 551             }
 552         }
 553
 554       if (PREDICT_FALSE
 555           (icmp_type_is_error_message
 556            (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags)))
 557         {
 558           b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
 559           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 560           goto out;
 561         }
 562
 563       next0 = slow_path (nm, b0, ip0, *addr, *port, *fib_index, *proto, &s0,
 564                          node, next0, thread_index, vlib_time_now (vm));
 565
 566       if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
 567         goto out;
 568
 569       if (!s0)
 570         {
 571           *dont_translate = 1;
 572           goto out;
 573         }
 574     }
 575   else
 576     {
 577       if (PREDICT_FALSE
 578           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 579            ICMP4_echo_request
 580            && vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 581            ICMP4_echo_reply
 582            && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 583                                            reass.icmp_type_or_tcp_flags)))
 584         {
 585           b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
 586           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 587           goto out;
 588         }
 589
 590       s0 = pool_elt_at_index (tnm->sessions,
 591                               nat_value_get_session_index (&value0));
 592     }
 593
 594 out:
 595   if (s0)
 596     {
 597       *addr = s0->out2in.addr;
 598       *port = s0->out2in.port;
 599       *fib_index = s0->out2in.fib_index;
 600     }
 601   if (p_s0)
 602     *p_s0 = s0;
 603   return next0;
 604 }
 605 #endif
 606
 607 #ifndef CLIB_MARCH_VARIANT
 608 u32
 609 nat44_ei_icmp_match_in2out_fast (vlib_node_runtime_t *node, u32 thread_index,
 610                                  vlib_buffer_t *b0, ip4_header_t *ip0,
 611                                  ip4_address_t *addr, u16 *port,
 612                                  u32 *fib_index, nat_protocol_t *proto,
 613                                  nat44_ei_session_t **s0, u8 *dont_translate)
 614 {
 615   u32 sw_if_index0;
 616   u8 is_addr_only;
 617   u32 next0 = ~0;
 618   int err;
 619   *dont_translate = 0;
 620
 621   sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 622   *fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 623
 624   err = icmp_get_key (b0, ip0, addr, port, proto);
 625   if (err != -1)
 626     {
 627       b0->error = node->errors[err];
 628       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 629       goto out;
 630     }
 631
 632   ip4_address_t sm_addr;
 633   u16 sm_port;
 634   u32 sm_fib_index;
 635
 636   if (nat44_ei_static_mapping_match (*addr, *port, *fib_index, *proto,
 637                                      &sm_addr, &sm_port, &sm_fib_index, 0,
 638                                      &is_addr_only, 0))
 639     {
 640       if (PREDICT_FALSE (nat44_ei_not_translate_fast (
 641             node, sw_if_index0, ip0, IP_PROTOCOL_ICMP, *fib_index)))
 642         {
 643           *dont_translate = 1;
 644           goto out;
 645         }
 646
 647       if (icmp_type_is_error_message
 648           (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags))
 649         {
 650           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 651           goto out;
 652         }
 653
 654       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_NO_TRANSLATION];
 655       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 656       goto out;
 657     }
 658
 659   if (PREDICT_FALSE
 660       (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags != ICMP4_echo_request
 661        && (vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags !=
 662            ICMP4_echo_reply || !is_addr_only)
 663        && !icmp_type_is_error_message (vnet_buffer (b0)->ip.
 664                                        reass.icmp_type_or_tcp_flags)))
 665     {
 666       b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_BAD_ICMP_TYPE];
 667       next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 668       goto out;
 669     }
 670
 671 out:
 672   return next0;
 673 }
 674 #endif
 675
 676 u32 nat44_ei_icmp_in2out (vlib_buffer_t *b0, ip4_header_t *ip0,
 677                           icmp46_header_t *icmp0, u32 sw_if_index0,
 678                           u32 rx_fib_index0, vlib_node_runtime_t *node,
 679                           u32 next0, u32 thread_index,
 680                           nat44_ei_session_t **p_s0);
 681
 682 #ifndef CLIB_MARCH_VARIANT
 683 u32
 684 nat44_ei_icmp_in2out (vlib_buffer_t *b0, ip4_header_t *ip0,
 685                       icmp46_header_t *icmp0, u32 sw_if_index0,
 686                       u32 rx_fib_index0, vlib_node_runtime_t *node, u32 next0,
 687                       u32 thread_index, nat44_ei_session_t **p_s0)
 688 {
 689   nat44_ei_main_t *nm = &nat44_ei_main;
 690   vlib_main_t *vm = vlib_get_main ();
 691   ip4_address_t addr;
 692   u16 port;
 693   u32 fib_index;
 694   nat_protocol_t proto;
 695   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 696   ip4_header_t *inner_ip0;
 697   void *l4_header = 0;
 698   icmp46_header_t *inner_icmp0;
 699   u8 dont_translate;
 700   u32 new_addr0, old_addr0;
 701   u16 old_id0, new_id0;
 702   u16 old_checksum0, new_checksum0;
 703   ip_csum_t sum0;
 704   u16 checksum0;
 705   u32 next0_tmp;
 706   u32 required_thread_index = thread_index;
 707
 708   echo0 = (icmp_echo_header_t *) (icmp0 + 1);
 709
 710   if (PREDICT_TRUE (nm->pat))
 711     {
 712       next0_tmp = nat44_ei_icmp_match_in2out_slow (
 713         node, thread_index, b0, ip0, &addr, &port, &fib_index, &proto, p_s0,
 714         &dont_translate);
 715     }
 716   else
 717     {
 718       next0_tmp = nat44_ei_icmp_match_in2out_fast (
 719         node, thread_index, b0, ip0, &addr, &port, &fib_index, &proto, p_s0,
 720         &dont_translate);
 721     }
 722
 723   if (next0_tmp != ~0)
 724     next0 = next0_tmp;
 725   if (next0 == NAT44_EI_IN2OUT_NEXT_DROP || dont_translate)
 726     goto out;
 727
 728   if (PREDICT_TRUE (!ip4_is_fragment (ip0)))
 729     {
 730       sum0 =
 731         ip_incremental_checksum_buffer (vm, b0,
 732                                         (u8 *) icmp0 -
 733                                         (u8 *) vlib_buffer_get_current (b0),
 734                                         ntohs (ip0->length) -
 735                                         ip4_header_bytes (ip0), 0);
 736       checksum0 = ~ip_csum_fold (sum0);
 737       if (PREDICT_FALSE (checksum0 != 0 && checksum0 != 0xffff))
 738         {
 739           next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 740           goto out;
 741         }
 742     }
 743
 744   old_addr0 = ip0->src_address.as_u32;
 745   new_addr0 = ip0->src_address.as_u32 = addr.as_u32;
 746
 747   sum0 = ip0->checksum;
 748   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 749                          src_address /* changed member */ );
 750   ip0->checksum = ip_csum_fold (sum0);
 751
 752   if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
 753     {
 754       if (icmp0->checksum == 0)
 755         icmp0->checksum = 0xffff;
 756
 757       if (!icmp_type_is_error_message (icmp0->type))
 758         {
 759           new_id0 = port;
 760           if (PREDICT_FALSE (new_id0 != echo0->identifier))
 761             {
 762               old_id0 = echo0->identifier;
 763               new_id0 = port;
 764               echo0->identifier = new_id0;
 765
 766               sum0 = icmp0->checksum;
 767               sum0 =
 768                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 769                                 identifier);
 770               icmp0->checksum = ip_csum_fold (sum0);
 771             }
 772         }
 773       else
 774         {
 775           inner_ip0 = (ip4_header_t *) (echo0 + 1);
 776           l4_header = ip4_next_header (inner_ip0);
 777
 778           if (!ip4_header_checksum_is_valid (inner_ip0))
 779             {
 780               next0 = NAT44_EI_IN2OUT_NEXT_DROP;
 781               goto out;
 782             }
 783
 784           /* update inner destination IP address */
 785           old_addr0 = inner_ip0->dst_address.as_u32;
 786           inner_ip0->dst_address = addr;
 787           new_addr0 = inner_ip0->dst_address.as_u32;
 788           sum0 = icmp0->checksum;
 789           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 790                                  dst_address /* changed member */ );
 791           icmp0->checksum = ip_csum_fold (sum0);
 792
 793           /* update inner IP header checksum */
 794           old_checksum0 = inner_ip0->checksum;
 795           sum0 = inner_ip0->checksum;
 796           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 797                                  dst_address /* changed member */ );
 798           inner_ip0->checksum = ip_csum_fold (sum0);
 799           new_checksum0 = inner_ip0->checksum;
 800           sum0 = icmp0->checksum;
 801           sum0 =
 802             ip_csum_update (sum0, old_checksum0, new_checksum0, ip4_header_t,
 803                             checksum);
 804           icmp0->checksum = ip_csum_fold (sum0);
 805
 806           switch (proto)
 807             {
 808             case NAT_PROTOCOL_ICMP:
 809               inner_icmp0 = (icmp46_header_t *) l4_header;
 810               inner_echo0 = (icmp_echo_header_t *) (inner_icmp0 + 1);
 811
 812               old_id0 = inner_echo0->identifier;
 813               new_id0 = port;
 814               inner_echo0->identifier = new_id0;
 815
 816               sum0 = icmp0->checksum;
 817               sum0 =
 818                 ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 819                                 identifier);
 820               icmp0->checksum = ip_csum_fold (sum0);
 821               break;
 822             case NAT_PROTOCOL_UDP:
 823             case NAT_PROTOCOL_TCP:
 824               old_id0 = ((tcp_udp_header_t *) l4_header)->dst_port;
 825               new_id0 = port;
 826               ((tcp_udp_header_t *) l4_header)->dst_port = new_id0;
 827
 828               sum0 = icmp0->checksum;
 829               sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 830                                      dst_port);
 831               icmp0->checksum = ip_csum_fold (sum0);
 832               break;
 833             default:
 834               ASSERT (0);
 835             }
 836         }
 837     }
 838
 839   if (vnet_buffer (b0)->sw_if_index[VLIB_TX] == ~0)
 840     {
 841       if (0 != nat44_ei_icmp_hairpinning (nm, b0, thread_index, ip0, icmp0,
 842                                           &required_thread_index))
 843         vnet_buffer (b0)->sw_if_index[VLIB_TX] = fib_index;
 844       if (thread_index != required_thread_index)
 845         {
 846           vnet_buffer (b0)->snat.required_thread_index = required_thread_index;
 847           next0 = NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF;
 848         }
 849     }
 850
 851 out:
 852   return next0;
 853 }
 854 #endif
 855
 856 static_always_inline u32
 857 nat44_ei_icmp_in2out_slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0,
 858                                 ip4_header_t *ip0, icmp46_header_t *icmp0,
 859                                 u32 sw_if_index0, u32 rx_fib_index0,
 860                                 vlib_node_runtime_t *node, u32 next0, f64 now,
 861                                 u32 thread_index, nat44_ei_session_t **p_s0)
 862 {
 863   vlib_main_t *vm = vlib_get_main ();
 864
 865   next0 = nat44_ei_icmp_in2out (b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
 866                                 node, next0, thread_index, p_s0);
 867   nat44_ei_session_t *s0 = *p_s0;
 868   if (PREDICT_TRUE (next0 != NAT44_EI_IN2OUT_NEXT_DROP && s0))
 869     {
 870       /* Accounting */
 871       nat44_ei_session_update_counters (
 872         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
 873       /* Per-user LRU list maintenance */
 874       nat44_ei_session_update_lru (nm, s0, thread_index);
 875     }
 876   return next0;
 877 }
 878
 879 static int
 880 nat_in2out_sm_unknown_proto (nat44_ei_main_t *nm, vlib_buffer_t *b,
 881                              ip4_header_t *ip, u32 rx_fib_index)
 882 {
 883   clib_bihash_kv_8_8_t kv, value;
 884   nat44_ei_static_mapping_t *m;
 885   u32 old_addr, new_addr;
 886   ip_csum_t sum;
 887
 888   init_nat_k (&kv, ip->src_address, 0, rx_fib_index, 0);
 889   if (clib_bihash_search_8_8 (&nm->static_mapping_by_local, &kv, &value))
 890     return 1;
 891
 892   m = pool_elt_at_index (nm->static_mappings, value.value);
 893
 894   old_addr = ip->src_address.as_u32;
 895   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
 896   sum = ip->checksum;
 897   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
 898   ip->checksum = ip_csum_fold (sum);
 899
 900
 901   /* Hairpinning */
 902   if (vnet_buffer (b)->sw_if_index[VLIB_TX] == ~0)
 903     {
 904       vnet_buffer (b)->sw_if_index[VLIB_TX] = m->fib_index;
 905       nat44_ei_hairpinning_sm_unknown_proto (nm, b, ip);
 906     }
 907
 908   return 0;
 909 }
 910
 911 static inline uword
 912 nat44_ei_in2out_node_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
 913                                 vlib_frame_t *frame, int is_slow_path,
 914                                 int is_output_feature)
 915 {
 916   u32 n_left_from, *from;
 917   nat44_ei_main_t *nm = &nat44_ei_main;
 918   f64 now = vlib_time_now (vm);
 919   u32 thread_index = vm->thread_index;
 920
 921   from = vlib_frame_vector_args (frame);
 922   n_left_from = frame->n_vectors;
 923
 924   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 925   u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
 926   vlib_get_buffers (vm, from, b, n_left_from);
 927
 928   while (n_left_from >= 2)
 929     {
 930       vlib_buffer_t *b0, *b1;
 931       u32 next0, next1;
 932       u32 sw_if_index0, sw_if_index1;
 933       ip4_header_t *ip0, *ip1;
 934       ip_csum_t sum0, sum1;
 935       u32 new_addr0, old_addr0, new_addr1, old_addr1;
 936       u16 old_port0, new_port0, old_port1, new_port1;
 937       udp_header_t *udp0, *udp1;
 938       tcp_header_t *tcp0, *tcp1;
 939       icmp46_header_t *icmp0, *icmp1;
 940       u32 rx_fib_index0, rx_fib_index1;
 941       u32 proto0, proto1;
 942       nat44_ei_session_t *s0 = 0, *s1 = 0;
 943       clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
 944       u32 iph_offset0 = 0, iph_offset1 = 0;
 945
 946       b0 = *b;
 947       b++;
 948       b1 = *b;
 949       b++;
 950
 951       /* Prefetch next iteration. */
 952       if (PREDICT_TRUE (n_left_from >= 4))
 953         {
 954           vlib_buffer_t *p2, *p3;
 955
 956           p2 = *b;
 957           p3 = *(b + 1);
 958
 959           vlib_prefetch_buffer_header (p2, LOAD);
 960           vlib_prefetch_buffer_header (p3, LOAD);
 961
 962           CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
 963           CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
 964         }
 965
 966       if (is_output_feature)
 967         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
 968
 969       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
 970                               iph_offset0);
 971
 972       udp0 = ip4_next_header (ip0);
 973       tcp0 = (tcp_header_t *) udp0;
 974       icmp0 = (icmp46_header_t *) udp0;
 975
 976       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 977       rx_fib_index0 =
 978         vec_elt (nm->ip4_main->fib_index_by_sw_if_index, sw_if_index0);
 979
 980       next0 = next1 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
 981
 982       if (PREDICT_FALSE (ip0->ttl == 1))
 983         {
 984           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
 985           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
 986                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
 987                                        0);
 988           next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
 989           goto trace00;
 990         }
 991
 992       proto0 = ip_proto_to_nat_proto (ip0->protocol);
 993
 994       /* Next configured feature, probably ip4-lookup */
 995       if (is_slow_path)
 996         {
 997           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
 998             {
 999               if (nat_in2out_sm_unknown_proto (nm, b0, ip0, rx_fib_index0))
1000                 {
1001                   next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1002                   b0->error =
1003                     node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1004                 }
1005               vlib_increment_simple_counter (
1006                 is_slow_path ? &nm->counters.slowpath.in2out.other :
1007                                &nm->counters.fastpath.in2out.other,
1008                 thread_index, sw_if_index0, 1);
1009               goto trace00;
1010             }
1011
1012           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1013             {
1014               next0 = nat44_ei_icmp_in2out_slow_path (
1015                 nm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, next0,
1016                 now, thread_index, &s0);
1017               vlib_increment_simple_counter (
1018                 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1019                                &nm->counters.fastpath.in2out.icmp,
1020                 thread_index, sw_if_index0, 1);
1021               goto trace00;
1022             }
1023         }
1024       else
1025         {
1026           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1027             {
1028               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1029               goto trace00;
1030             }
1031
1032           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1033             {
1034               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1035               goto trace00;
1036             }
1037         }
1038
1039       init_nat_k (&kv0, ip0->src_address,
1040                   vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1041                   proto0);
1042       if (PREDICT_FALSE (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0) !=
1043                          0))
1044         {
1045           if (is_slow_path)
1046             {
1047               if (is_output_feature)
1048                 {
1049                   if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1050                         nm, ip0, proto0,
1051                         vnet_buffer (b0)->ip.reass.l4_src_port,
1052                         vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1053                         sw_if_index0)))
1054                     goto trace00;
1055
1056                   /*
1057                    * Send DHCP packets to the ipv4 stack, or we won't
1058                    * be able to use dhcp client on the outside interface
1059                    */
1060                   if (PREDICT_FALSE
1061                       (proto0 == NAT_PROTOCOL_UDP
1062                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1063                            clib_host_to_net_u16
1064                            (UDP_DST_PORT_dhcp_to_server))
1065                        && ip0->dst_address.as_u32 == 0xffffffff))
1066                     goto trace00;
1067                 }
1068               else
1069                 {
1070                   if (PREDICT_FALSE (nat44_ei_not_translate (
1071                         nm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1072                         thread_index)))
1073                     goto trace00;
1074                 }
1075
1076               next0 = slow_path (nm, b0, ip0, ip0->src_address,
1077                                  vnet_buffer (b0)->ip.reass.l4_src_port,
1078                                  rx_fib_index0, proto0, &s0, node, next0,
1079                                  thread_index, now);
1080               if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
1081                 goto trace00;
1082
1083               if (PREDICT_FALSE (!s0))
1084                 goto trace00;
1085             }
1086           else
1087             {
1088               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1089               goto trace00;
1090             }
1091         }
1092       else
1093         s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1094                                 nat_value_get_session_index (&value0));
1095
1096       b0->flags |= VNET_BUFFER_F_IS_NATED;
1097
1098       old_addr0 = ip0->src_address.as_u32;
1099       ip0->src_address = s0->out2in.addr;
1100       new_addr0 = ip0->src_address.as_u32;
1101       if (!is_output_feature)
1102         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1103
1104       sum0 = ip0->checksum;
1105       sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1106                              ip4_header_t, src_address /* changed member */ );
1107       ip0->checksum = ip_csum_fold (sum0);
1108
1109
1110       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1111         {
1112           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1113             {
1114               old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1115               new_port0 = udp0->src_port = s0->out2in.port;
1116               sum0 = tcp0->checksum;
1117               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1118                                      ip4_header_t,
1119                                      dst_address /* changed member */ );
1120               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1121                                      ip4_header_t /* cheat */ ,
1122                                      length /* changed member */ );
1123               mss_clamping (nm->mss_clamping, tcp0, &sum0);
1124               tcp0->checksum = ip_csum_fold (sum0);
1125             }
1126           vlib_increment_simple_counter (is_slow_path ?
1127                                            &nm->counters.slowpath.in2out.tcp :
1128                                            &nm->counters.fastpath.in2out.tcp,
1129                                          thread_index, sw_if_index0, 1);
1130         }
1131       else
1132         {
1133           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1134             {
1135               udp0->src_port = s0->out2in.port;
1136               if (PREDICT_FALSE (udp0->checksum))
1137                 {
1138                   old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1139                   new_port0 = udp0->src_port;
1140                   sum0 = udp0->checksum;
1141                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t, dst_address  /* changed member */
1142                     );
1143                   sum0 =
1144                     ip_csum_update (sum0, old_port0, new_port0,
1145                                     ip4_header_t /* cheat */ ,
1146                                     length /* changed member */ );
1147                   udp0->checksum = ip_csum_fold (sum0);
1148                 }
1149             }
1150           vlib_increment_simple_counter (is_slow_path ?
1151                                            &nm->counters.slowpath.in2out.udp :
1152                                            &nm->counters.fastpath.in2out.udp,
1153                                          thread_index, sw_if_index0, 1);
1154         }
1155
1156       /* Accounting */
1157       nat44_ei_session_update_counters (
1158         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1159       /* Per-user LRU list maintenance */
1160       nat44_ei_session_update_lru (nm, s0, thread_index);
1161     trace00:
1162
1163       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1164                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1165         {
1166           nat44_ei_in2out_trace_t *t =
1167             vlib_add_trace (vm, node, b0, sizeof (*t));
1168           t->is_slow_path = is_slow_path;
1169           t->sw_if_index = sw_if_index0;
1170           t->next_index = next0;
1171           t->session_index = ~0;
1172           if (s0)
1173             t->session_index = s0 - nm->per_thread_data[thread_index].sessions;
1174         }
1175
1176       if (next0 == NAT44_EI_IN2OUT_NEXT_DROP)
1177         {
1178           vlib_increment_simple_counter (
1179             is_slow_path ? &nm->counters.slowpath.in2out.drops :
1180                            &nm->counters.fastpath.in2out.drops,
1181             thread_index, sw_if_index0, 1);
1182         }
1183
1184       if (is_output_feature)
1185         iph_offset1 = vnet_buffer (b1)->ip.reass.save_rewrite_length;
1186
1187       ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1188                               iph_offset1);
1189
1190       udp1 = ip4_next_header (ip1);
1191       tcp1 = (tcp_header_t *) udp1;
1192       icmp1 = (icmp46_header_t *) udp1;
1193
1194       sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1195       rx_fib_index1 =
1196         vec_elt (nm->ip4_main->fib_index_by_sw_if_index, sw_if_index1);
1197
1198       if (PREDICT_FALSE (ip1->ttl == 1))
1199         {
1200           vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1201           icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1202                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1203                                        0);
1204           next1 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1205           goto trace01;
1206         }
1207
1208       proto1 = ip_proto_to_nat_proto (ip1->protocol);
1209
1210       /* Next configured feature, probably ip4-lookup */
1211       if (is_slow_path)
1212         {
1213           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1214             {
1215               if (nat_in2out_sm_unknown_proto (nm, b1, ip1, rx_fib_index1))
1216                 {
1217                   next1 = NAT44_EI_IN2OUT_NEXT_DROP;
1218                   b1->error =
1219                     node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1220                 }
1221               vlib_increment_simple_counter (
1222                 is_slow_path ? &nm->counters.slowpath.in2out.other :
1223                                &nm->counters.fastpath.in2out.other,
1224                 thread_index, sw_if_index1, 1);
1225               goto trace01;
1226             }
1227
1228           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1229             {
1230               next1 = nat44_ei_icmp_in2out_slow_path (
1231                 nm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, next1,
1232                 now, thread_index, &s1);
1233               vlib_increment_simple_counter (
1234                 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1235                                &nm->counters.fastpath.in2out.icmp,
1236                 thread_index, sw_if_index1, 1);
1237               goto trace01;
1238             }
1239         }
1240       else
1241         {
1242           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_OTHER))
1243             {
1244               next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1245               goto trace01;
1246             }
1247
1248           if (PREDICT_FALSE (proto1 == NAT_PROTOCOL_ICMP))
1249             {
1250               next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1251               goto trace01;
1252             }
1253         }
1254
1255       init_nat_k (&kv1, ip1->src_address,
1256                   vnet_buffer (b1)->ip.reass.l4_src_port, rx_fib_index1,
1257                   proto1);
1258       if (PREDICT_FALSE (clib_bihash_search_8_8 (&nm->in2out, &kv1, &value1) !=
1259                          0))
1260         {
1261           if (is_slow_path)
1262             {
1263               if (is_output_feature)
1264                 {
1265                   if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1266                         nm, ip1, proto1,
1267                         vnet_buffer (b1)->ip.reass.l4_src_port,
1268                         vnet_buffer (b1)->ip.reass.l4_dst_port, thread_index,
1269                         sw_if_index1)))
1270                     goto trace01;
1271
1272                   /*
1273                    * Send DHCP packets to the ipv4 stack, or we won't
1274                    * be able to use dhcp client on the outside interface
1275                    */
1276                   if (PREDICT_FALSE
1277                       (proto1 == NAT_PROTOCOL_UDP
1278                        && (vnet_buffer (b1)->ip.reass.l4_dst_port ==
1279                            clib_host_to_net_u16
1280                            (UDP_DST_PORT_dhcp_to_server))
1281                        && ip1->dst_address.as_u32 == 0xffffffff))
1282                     goto trace01;
1283                 }
1284               else
1285                 {
1286                   if (PREDICT_FALSE (nat44_ei_not_translate (
1287                         nm, node, sw_if_index1, ip1, proto1, rx_fib_index1,
1288                         thread_index)))
1289                     goto trace01;
1290                 }
1291
1292               next1 = slow_path (nm, b1, ip1, ip1->src_address,
1293                                  vnet_buffer (b1)->ip.reass.l4_src_port,
1294                                  rx_fib_index1, proto1, &s1, node, next1,
1295                                  thread_index, now);
1296               if (PREDICT_FALSE (next1 == NAT44_EI_IN2OUT_NEXT_DROP))
1297                 goto trace01;
1298
1299               if (PREDICT_FALSE (!s1))
1300                 goto trace01;
1301             }
1302           else
1303             {
1304               next1 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1305               goto trace01;
1306             }
1307         }
1308       else
1309         s1 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1310                                 nat_value_get_session_index (&value1));
1311
1312       b1->flags |= VNET_BUFFER_F_IS_NATED;
1313
1314       old_addr1 = ip1->src_address.as_u32;
1315       ip1->src_address = s1->out2in.addr;
1316       new_addr1 = ip1->src_address.as_u32;
1317       if (!is_output_feature)
1318         vnet_buffer (b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1319
1320       sum1 = ip1->checksum;
1321       sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1322                              ip4_header_t, src_address /* changed member */ );
1323       ip1->checksum = ip_csum_fold (sum1);
1324
1325       if (PREDICT_TRUE (proto1 == NAT_PROTOCOL_TCP))
1326         {
1327           if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1328             {
1329               old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1330               new_port1 = udp1->src_port = s1->out2in.port;
1331               sum1 = tcp1->checksum;
1332               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1333                                      ip4_header_t,
1334                                      dst_address /* changed member */ );
1335               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1336                                      ip4_header_t /* cheat */ ,
1337                                      length /* changed member */ );
1338               mss_clamping (nm->mss_clamping, tcp1, &sum1);
1339               tcp1->checksum = ip_csum_fold (sum1);
1340             }
1341           vlib_increment_simple_counter (is_slow_path ?
1342                                            &nm->counters.slowpath.in2out.tcp :
1343                                            &nm->counters.fastpath.in2out.tcp,
1344                                          thread_index, sw_if_index1, 1);
1345         }
1346       else
1347         {
1348           if (!vnet_buffer (b1)->ip.reass.is_non_first_fragment)
1349             {
1350               udp1->src_port = s1->out2in.port;
1351               if (PREDICT_FALSE (udp1->checksum))
1352                 {
1353                   old_port1 = vnet_buffer (b1)->ip.reass.l4_src_port;
1354                   new_port1 = udp1->src_port;
1355                   sum1 = udp1->checksum;
1356                   sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t, dst_address  /* changed member */
1357                     );
1358                   sum1 =
1359                     ip_csum_update (sum1, old_port1, new_port1,
1360                                     ip4_header_t /* cheat */ ,
1361                                     length /* changed member */ );
1362                   udp1->checksum = ip_csum_fold (sum1);
1363                 }
1364             }
1365           vlib_increment_simple_counter (is_slow_path ?
1366                                            &nm->counters.slowpath.in2out.udp :
1367                                            &nm->counters.fastpath.in2out.udp,
1368                                          thread_index, sw_if_index1, 1);
1369         }
1370
1371       /* Accounting */
1372       nat44_ei_session_update_counters (
1373         s1, now, vlib_buffer_length_in_chain (vm, b1), thread_index);
1374       /* Per-user LRU list maintenance */
1375       nat44_ei_session_update_lru (nm, s1, thread_index);
1376     trace01:
1377
1378       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1379                          && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1380         {
1381           nat44_ei_in2out_trace_t *t =
1382             vlib_add_trace (vm, node, b1, sizeof (*t));
1383           t->sw_if_index = sw_if_index1;
1384           t->next_index = next1;
1385           t->session_index = ~0;
1386           if (s1)
1387             t->session_index = s1 - nm->per_thread_data[thread_index].sessions;
1388         }
1389
1390       if (next1 == NAT44_EI_IN2OUT_NEXT_DROP)
1391         {
1392           vlib_increment_simple_counter (
1393             is_slow_path ? &nm->counters.slowpath.in2out.drops :
1394                            &nm->counters.fastpath.in2out.drops,
1395             thread_index, sw_if_index1, 1);
1396         }
1397
1398       n_left_from -= 2;
1399       next[0] = next0;
1400       next[1] = next1;
1401       next += 2;
1402     }
1403
1404   while (n_left_from > 0)
1405     {
1406       vlib_buffer_t *b0;
1407       u32 next0;
1408       u32 sw_if_index0;
1409       ip4_header_t *ip0;
1410       ip_csum_t sum0;
1411       u32 new_addr0, old_addr0;
1412       u16 old_port0, new_port0;
1413       udp_header_t *udp0;
1414       tcp_header_t *tcp0;
1415       icmp46_header_t *icmp0;
1416       u32 rx_fib_index0;
1417       u32 proto0;
1418       nat44_ei_session_t *s0 = 0;
1419       clib_bihash_kv_8_8_t kv0, value0;
1420       u32 iph_offset0 = 0;
1421
1422       b0 = *b;
1423       b++;
1424       next0 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
1425
1426       if (is_output_feature)
1427         iph_offset0 = vnet_buffer (b0)->ip.reass.save_rewrite_length;
1428
1429       ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1430                               iph_offset0);
1431
1432       udp0 = ip4_next_header (ip0);
1433       tcp0 = (tcp_header_t *) udp0;
1434       icmp0 = (icmp46_header_t *) udp0;
1435
1436       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1437       rx_fib_index0 =
1438         vec_elt (nm->ip4_main->fib_index_by_sw_if_index, sw_if_index0);
1439
1440       if (PREDICT_FALSE (ip0->ttl == 1))
1441         {
1442           vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1443           icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1444                                        ICMP4_time_exceeded_ttl_exceeded_in_transit,
1445                                        0);
1446           next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1447           goto trace0;
1448         }
1449
1450       proto0 = ip_proto_to_nat_proto (ip0->protocol);
1451
1452       /* Next configured feature, probably ip4-lookup */
1453       if (is_slow_path)
1454         {
1455           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1456             {
1457               if (nat_in2out_sm_unknown_proto (nm, b0, ip0, rx_fib_index0))
1458                 {
1459                   next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1460                   b0->error =
1461                     node->errors[NAT44_EI_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1462                 }
1463               vlib_increment_simple_counter (
1464                 is_slow_path ? &nm->counters.slowpath.in2out.other :
1465                                &nm->counters.fastpath.in2out.other,
1466                 thread_index, sw_if_index0, 1);
1467               goto trace0;
1468             }
1469
1470           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1471             {
1472               next0 = nat44_ei_icmp_in2out_slow_path (
1473                 nm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, next0,
1474                 now, thread_index, &s0);
1475               vlib_increment_simple_counter (
1476                 is_slow_path ? &nm->counters.slowpath.in2out.icmp :
1477                                &nm->counters.fastpath.in2out.icmp,
1478                 thread_index, sw_if_index0, 1);
1479               goto trace0;
1480             }
1481         }
1482       else
1483         {
1484           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1485             {
1486               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1487               goto trace0;
1488             }
1489
1490           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1491             {
1492               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1493               goto trace0;
1494             }
1495         }
1496
1497       init_nat_k (&kv0, ip0->src_address,
1498                   vnet_buffer (b0)->ip.reass.l4_src_port, rx_fib_index0,
1499                   proto0);
1500
1501       if (clib_bihash_search_8_8 (&nm->in2out, &kv0, &value0))
1502         {
1503           if (is_slow_path)
1504             {
1505               if (is_output_feature)
1506                 {
1507                   if (PREDICT_FALSE (nat44_ei_not_translate_output_feature (
1508                         nm, ip0, proto0,
1509                         vnet_buffer (b0)->ip.reass.l4_src_port,
1510                         vnet_buffer (b0)->ip.reass.l4_dst_port, thread_index,
1511                         sw_if_index0)))
1512                     goto trace0;
1513
1514                   /*
1515                    * Send DHCP packets to the ipv4 stack, or we won't
1516                    * be able to use dhcp client on the outside interface
1517                    */
1518                   if (PREDICT_FALSE
1519                       (proto0 == NAT_PROTOCOL_UDP
1520                        && (vnet_buffer (b0)->ip.reass.l4_dst_port ==
1521                            clib_host_to_net_u16
1522                            (UDP_DST_PORT_dhcp_to_server))
1523                        && ip0->dst_address.as_u32 == 0xffffffff))
1524                     goto trace0;
1525                 }
1526               else
1527                 {
1528                   if (PREDICT_FALSE (nat44_ei_not_translate (
1529                         nm, node, sw_if_index0, ip0, proto0, rx_fib_index0,
1530                         thread_index)))
1531                     goto trace0;
1532                 }
1533
1534               next0 = slow_path (nm, b0, ip0, ip0->src_address,
1535                                  vnet_buffer (b0)->ip.reass.l4_src_port,
1536                                  rx_fib_index0, proto0, &s0, node, next0,
1537                                  thread_index, now);
1538
1539               if (PREDICT_FALSE (next0 == NAT44_EI_IN2OUT_NEXT_DROP))
1540                 goto trace0;
1541
1542               if (PREDICT_FALSE (!s0))
1543                 goto trace0;
1544             }
1545           else
1546             {
1547               next0 = NAT44_EI_IN2OUT_NEXT_SLOW_PATH;
1548               goto trace0;
1549             }
1550         }
1551       else
1552         s0 = pool_elt_at_index (nm->per_thread_data[thread_index].sessions,
1553                                 nat_value_get_session_index (&value0));
1554
1555       b0->flags |= VNET_BUFFER_F_IS_NATED;
1556
1557       old_addr0 = ip0->src_address.as_u32;
1558       ip0->src_address = s0->out2in.addr;
1559       new_addr0 = ip0->src_address.as_u32;
1560       if (!is_output_feature)
1561         vnet_buffer (b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1562
1563       sum0 = ip0->checksum;
1564       sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1565                              ip4_header_t, src_address /* changed member */ );
1566       ip0->checksum = ip_csum_fold (sum0);
1567
1568       if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1569         {
1570           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1571             {
1572               old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1573               new_port0 = udp0->src_port = s0->out2in.port;
1574               sum0 = tcp0->checksum;
1575               sum0 =
1576                 ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1577                                 dst_address /* changed member */ );
1578               sum0 =
1579                 ip_csum_update (sum0, old_port0, new_port0,
1580                                 ip4_header_t /* cheat */ ,
1581                                 length /* changed member */ );
1582               mss_clamping (nm->mss_clamping, tcp0, &sum0);
1583               tcp0->checksum = ip_csum_fold (sum0);
1584             }
1585           vlib_increment_simple_counter (is_slow_path ?
1586                                            &nm->counters.slowpath.in2out.tcp :
1587                                            &nm->counters.fastpath.in2out.tcp,
1588                                          thread_index, sw_if_index0, 1);
1589         }
1590       else
1591         {
1592           if (!vnet_buffer (b0)->ip.reass.is_non_first_fragment)
1593             {
1594               udp0->src_port = s0->out2in.port;
1595               if (PREDICT_FALSE (udp0->checksum))
1596                 {
1597                   old_port0 = vnet_buffer (b0)->ip.reass.l4_src_port;
1598                   new_port0 = udp0->src_port;
1599                   sum0 = udp0->checksum;
1600                   sum0 =
1601                     ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
1602                                     dst_address /* changed member */ );
1603                   sum0 =
1604                     ip_csum_update (sum0, old_port0, new_port0,
1605                                     ip4_header_t /* cheat */ ,
1606                                     length /* changed member */ );
1607                   udp0->checksum = ip_csum_fold (sum0);
1608                 }
1609             }
1610           vlib_increment_simple_counter (is_slow_path ?
1611                                            &nm->counters.slowpath.in2out.udp :
1612                                            &nm->counters.fastpath.in2out.udp,
1613                                          thread_index, sw_if_index0, 1);
1614         }
1615
1616       /* Accounting */
1617       nat44_ei_session_update_counters (
1618         s0, now, vlib_buffer_length_in_chain (vm, b0), thread_index);
1619       /* Per-user LRU list maintenance */
1620       nat44_ei_session_update_lru (nm, s0, thread_index);
1621
1622     trace0:
1623       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1624                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1625         {
1626           nat44_ei_in2out_trace_t *t =
1627             vlib_add_trace (vm, node, b0, sizeof (*t));
1628           t->is_slow_path = is_slow_path;
1629           t->sw_if_index = sw_if_index0;
1630           t->next_index = next0;
1631           t->session_index = ~0;
1632           if (s0)
1633             t->session_index = s0 - nm->per_thread_data[thread_index].sessions;
1634         }
1635
1636       if (next0 == NAT44_EI_IN2OUT_NEXT_DROP)
1637         {
1638           vlib_increment_simple_counter (
1639             is_slow_path ? &nm->counters.slowpath.in2out.drops :
1640                            &nm->counters.fastpath.in2out.drops,
1641             thread_index, sw_if_index0, 1);
1642         }
1643
1644       n_left_from--;
1645       next[0] = next0;
1646       next++;
1647     }
1648
1649   vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
1650                                frame->n_vectors);
1651   return frame->n_vectors;
1652 }
1653
1654 VLIB_NODE_FN (nat44_ei_in2out_node)
1655 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1656 {
1657   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */,
1658                                          0);
1659 }
1660
1661 VLIB_REGISTER_NODE (nat44_ei_in2out_node) = {
1662   .name = "nat44-ei-in2out",
1663   .vector_size = sizeof (u32),
1664   .format_trace = format_nat44_ei_in2out_trace,
1665   .type = VLIB_NODE_TYPE_INTERNAL,
1666
1667   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1668   .error_strings = nat44_ei_in2out_error_strings,
1669
1670   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1671
1672   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1673
1674   /* edit / add dispositions here */
1675   .next_nodes = {
1676     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1677     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1678     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
1679     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1680     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
1681   },
1682 };
1683
1684 VLIB_NODE_FN (nat44_ei_in2out_output_node)
1685 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1686 {
1687   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */,
1688                                          1);
1689 }
1690
1691 VLIB_REGISTER_NODE (nat44_ei_in2out_output_node) = {
1692   .name = "nat44-ei-in2out-output",
1693   .vector_size = sizeof (u32),
1694   .format_trace = format_nat44_ei_in2out_trace,
1695   .type = VLIB_NODE_TYPE_INTERNAL,
1696
1697   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1698   .error_strings = nat44_ei_in2out_error_strings,
1699
1700   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1701
1702   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1703
1704   /* edit / add dispositions here */
1705   .next_nodes = {
1706     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1707     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "interface-output",
1708     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-output-slowpath",
1709     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1710     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-interface-output",
1711   },
1712 };
1713
1714 VLIB_NODE_FN (nat44_ei_in2out_slowpath_node)
1715 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1716 {
1717   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */,
1718                                          0);
1719 }
1720
1721 VLIB_REGISTER_NODE (nat44_ei_in2out_slowpath_node) = {
1722   .name = "nat44-ei-in2out-slowpath",
1723   .vector_size = sizeof (u32),
1724   .format_trace = format_nat44_ei_in2out_trace,
1725   .type = VLIB_NODE_TYPE_INTERNAL,
1726
1727   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1728   .error_strings = nat44_ei_in2out_error_strings,
1729
1730   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1731
1732   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1733
1734   /* edit / add dispositions here */
1735   .next_nodes = {
1736     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1737     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1738     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
1739     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1740     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
1741   },
1742 };
1743
1744 VLIB_NODE_FN (nat44_ei_in2out_output_slowpath_node)
1745 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1746 {
1747   return nat44_ei_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */,
1748                                          1);
1749 }
1750
1751 VLIB_REGISTER_NODE (nat44_ei_in2out_output_slowpath_node) = {
1752   .name = "nat44-ei-in2out-output-slowpath",
1753   .vector_size = sizeof (u32),
1754   .format_trace = format_nat44_ei_in2out_trace,
1755   .type = VLIB_NODE_TYPE_INTERNAL,
1756
1757   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1758   .error_strings = nat44_ei_in2out_error_strings,
1759
1760   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1761
1762   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1763
1764   /* edit / add dispositions here */
1765   .next_nodes = {
1766     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1767     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "interface-output",
1768     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-output-slowpath",
1769     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1770     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-interface-output",
1771   },
1772 };
1773
1774 VLIB_NODE_FN (nat44_ei_in2out_fast_node)
1775 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1776 {
1777   u32 n_left_from, *from, *to_next;
1778   u32 thread_index = vm->thread_index;
1779   nat44_ei_in2out_next_t next_index;
1780   nat44_ei_main_t *nm = &nat44_ei_main;
1781   int is_hairpinning = 0;
1782
1783   from = vlib_frame_vector_args (frame);
1784   n_left_from = frame->n_vectors;
1785   next_index = node->cached_next_index;
1786
1787   while (n_left_from > 0)
1788     {
1789       u32 n_left_to_next;
1790
1791       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1792
1793       while (n_left_from > 0 && n_left_to_next > 0)
1794         {
1795           u32 bi0;
1796           vlib_buffer_t *b0;
1797           u32 next0;
1798           u32 sw_if_index0;
1799           ip4_header_t *ip0;
1800           ip_csum_t sum0;
1801           u32 new_addr0, old_addr0;
1802           u16 old_port0, new_port0;
1803           udp_header_t *udp0;
1804           tcp_header_t *tcp0;
1805           icmp46_header_t *icmp0;
1806           u32 proto0;
1807           u32 rx_fib_index0;
1808           ip4_address_t sm0_addr;
1809           u16 sm0_port;
1810           u32 sm0_fib_index;
1811           u32 required_thread_index = thread_index;
1812
1813           /* speculatively enqueue b0 to the current next frame */
1814           bi0 = from[0];
1815           to_next[0] = bi0;
1816           from += 1;
1817           to_next += 1;
1818           n_left_from -= 1;
1819           n_left_to_next -= 1;
1820
1821           b0 = vlib_get_buffer (vm, bi0);
1822           next0 = NAT44_EI_IN2OUT_NEXT_LOOKUP;
1823
1824           ip0 = vlib_buffer_get_current (b0);
1825           udp0 = ip4_next_header (ip0);
1826           tcp0 = (tcp_header_t *) udp0;
1827           icmp0 = (icmp46_header_t *) udp0;
1828
1829           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1830           rx_fib_index0 =
1831             ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
1832
1833           if (PREDICT_FALSE (ip0->ttl == 1))
1834             {
1835               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1836               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1837                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1838                                            0);
1839               next0 = NAT44_EI_IN2OUT_NEXT_ICMP_ERROR;
1840               goto trace0;
1841             }
1842
1843           proto0 = ip_proto_to_nat_proto (ip0->protocol);
1844
1845           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_OTHER))
1846             goto trace0;
1847
1848           if (PREDICT_FALSE (proto0 == NAT_PROTOCOL_ICMP))
1849             {
1850               next0 = nat44_ei_icmp_in2out (b0, ip0, icmp0, sw_if_index0,
1851                                             rx_fib_index0, node, next0, ~0, 0);
1852               goto trace0;
1853             }
1854
1855           if (nat44_ei_static_mapping_match (
1856                 ip0->src_address, udp0->src_port, rx_fib_index0, proto0,
1857                 &sm0_addr, &sm0_port, &sm0_fib_index, 0, 0, 0))
1858             {
1859               b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_NO_TRANSLATION];
1860               next0 = NAT44_EI_IN2OUT_NEXT_DROP;
1861               goto trace0;
1862             }
1863
1864           new_addr0 = sm0_addr.as_u32;
1865           new_port0 = sm0_port;
1866           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sm0_fib_index;
1867           old_addr0 = ip0->src_address.as_u32;
1868           ip0->src_address.as_u32 = new_addr0;
1869
1870           sum0 = ip0->checksum;
1871           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1872                                  ip4_header_t,
1873                                  src_address /* changed member */ );
1874           ip0->checksum = ip_csum_fold (sum0);
1875
1876           if (PREDICT_FALSE (new_port0 != udp0->dst_port))
1877             {
1878               old_port0 = udp0->src_port;
1879               udp0->src_port = new_port0;
1880
1881               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1882                 {
1883                   sum0 = tcp0->checksum;
1884                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1885                                          ip4_header_t,
1886                                          dst_address /* changed member */ );
1887                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1888                                          ip4_header_t /* cheat */ ,
1889                                          length /* changed member */ );
1890                   mss_clamping (nm->mss_clamping, tcp0, &sum0);
1891                   tcp0->checksum = ip_csum_fold (sum0);
1892                 }
1893               else if (udp0->checksum)
1894                 {
1895                   sum0 = udp0->checksum;
1896                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1897                                          ip4_header_t,
1898                                          dst_address /* changed member */ );
1899                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
1900                                          ip4_header_t /* cheat */ ,
1901                                          length /* changed member */ );
1902                   udp0->checksum = ip_csum_fold (sum0);
1903                 }
1904             }
1905           else
1906             {
1907               if (PREDICT_TRUE (proto0 == NAT_PROTOCOL_TCP))
1908                 {
1909                   sum0 = tcp0->checksum;
1910                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1911                                          ip4_header_t,
1912                                          dst_address /* changed member */ );
1913                   mss_clamping (nm->mss_clamping, tcp0, &sum0);
1914                   tcp0->checksum = ip_csum_fold (sum0);
1915                 }
1916               else if (udp0->checksum)
1917                 {
1918                   sum0 = udp0->checksum;
1919                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1920                                          ip4_header_t,
1921                                          dst_address /* changed member */ );
1922                   udp0->checksum = ip_csum_fold (sum0);
1923                 }
1924             }
1925
1926           /* Hairpinning */
1927           is_hairpinning = nat44_ei_hairpinning (
1928             vm, node, nm, thread_index, b0, ip0, udp0, tcp0, proto0,
1929             0 /* do_trace */, &required_thread_index);
1930
1931           if (thread_index != required_thread_index)
1932             {
1933               vnet_buffer (b0)->snat.required_thread_index =
1934                 required_thread_index;
1935               next0 = NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF;
1936             }
1937
1938         trace0:
1939           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
1940                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1941             {
1942               nat44_ei_in2out_trace_t *t =
1943                 vlib_add_trace (vm, node, b0, sizeof (*t));
1944               t->sw_if_index = sw_if_index0;
1945               t->next_index = next0;
1946               t->is_hairpinning = is_hairpinning;
1947             }
1948
1949           if (next0 != NAT44_EI_IN2OUT_NEXT_DROP)
1950             {
1951
1952               vlib_increment_simple_counter (
1953                 &nm->counters.fastpath.in2out.other, sw_if_index0,
1954                 vm->thread_index, 1);
1955             }
1956
1957           /* verify speculative enqueue, maybe switch current next frame */
1958           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1959                                            to_next, n_left_to_next,
1960                                            bi0, next0);
1961         }
1962
1963       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1964     }
1965
1966   return frame->n_vectors;
1967 }
1968
1969 VLIB_REGISTER_NODE (nat44_ei_in2out_fast_node) = {
1970   .name = "nat44-ei-in2out-fast",
1971   .vector_size = sizeof (u32),
1972   .format_trace = format_nat44_ei_in2out_fast_trace,
1973   .type = VLIB_NODE_TYPE_INTERNAL,
1974
1975   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
1976   .error_strings = nat44_ei_in2out_error_strings,
1977
1978   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
1979
1980   .n_next_nodes = NAT44_EI_IN2OUT_N_NEXT,
1981
1982   /* edit / add dispositions here */
1983   .next_nodes = {
1984     [NAT44_EI_IN2OUT_NEXT_DROP] = "error-drop",
1985     [NAT44_EI_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1986     [NAT44_EI_IN2OUT_NEXT_SLOW_PATH] = "nat44-ei-in2out-slowpath",
1987     [NAT44_EI_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1988     [NAT44_EI_IN2OUT_NEXT_HAIRPINNING_HANDOFF] = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
1989   },
1990 };
1991
1992 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_handoff_ip4_lookup_node)
1993 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1994 {
1995   return nat44_ei_hairpinning_handoff_fn_inline (
1996     vm, node, frame,
1997     nat44_ei_main.in2out_hairpinning_finish_ip4_lookup_node_fq_index);
1998 }
1999
2000 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_handoff_ip4_lookup_node) = {
2001   .name = "nat44-ei-in2out-hairpinning-handoff-ip4-lookup",
2002   .vector_size = sizeof (u32),
2003   .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2004   .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2005   .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2006
2007   .n_next_nodes = 1,
2008
2009   .next_nodes = {
2010     [0] = "error-drop",
2011   },
2012 };
2013
2014 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_handoff_interface_output_node)
2015 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2016 {
2017   return nat44_ei_hairpinning_handoff_fn_inline (
2018     vm, node, frame,
2019     nat44_ei_main.in2out_hairpinning_finish_interface_output_node_fq_index);
2020 }
2021
2022 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_handoff_interface_output_node) = {
2023   .name = "nat44-ei-in2out-hairpinning-handoff-interface-output",
2024   .vector_size = sizeof (u32),
2025   .n_errors = ARRAY_LEN(nat44_ei_hairpinning_handoff_error_strings),
2026   .error_strings = nat44_ei_hairpinning_handoff_error_strings,
2027   .format_trace = format_nat44_ei_hairpinning_handoff_trace,
2028
2029   .n_next_nodes = 1,
2030
2031   .next_nodes = {
2032     [0] = "error-drop",
2033   },
2034 };
2035
2036 static_always_inline int
2037 nat44_ei_in2out_hairpinning_finish_inline (vlib_main_t *vm,
2038                                            vlib_node_runtime_t *node,
2039                                            vlib_frame_t *frame)
2040 {
2041   u32 n_left_from, *from, *to_next;
2042   u32 thread_index = vm->thread_index;
2043   nat44_ei_in2out_next_t next_index;
2044   nat44_ei_main_t *nm = &nat44_ei_main;
2045   int is_hairpinning = 0;
2046
2047   from = vlib_frame_vector_args (frame);
2048   n_left_from = frame->n_vectors;
2049   next_index = node->cached_next_index;
2050
2051   while (n_left_from > 0)
2052     {
2053       u32 n_left_to_next;
2054
2055       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2056
2057       while (n_left_from > 0 && n_left_to_next > 0)
2058         {
2059           u32 bi0;
2060           vlib_buffer_t *b0;
2061           u32 next0;
2062           u32 sw_if_index0;
2063           ip4_header_t *ip0;
2064           udp_header_t *udp0;
2065           tcp_header_t *tcp0;
2066           icmp46_header_t *icmp0;
2067           u32 proto0;
2068           u32 required_thread_index = thread_index;
2069
2070           /* speculatively enqueue b0 to the current next frame */
2071           bi0 = from[0];
2072           to_next[0] = bi0;
2073           from += 1;
2074           to_next += 1;
2075           n_left_from -= 1;
2076           n_left_to_next -= 1;
2077
2078           b0 = vlib_get_buffer (vm, bi0);
2079           next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP;
2080
2081           ip0 = vlib_buffer_get_current (b0);
2082           udp0 = ip4_next_header (ip0);
2083           tcp0 = (tcp_header_t *) udp0;
2084           icmp0 = (icmp46_header_t *) udp0;
2085
2086           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2087           proto0 = ip_proto_to_nat_proto (ip0->protocol);
2088
2089           switch (proto0)
2090             {
2091             case NAT_PROTOCOL_TCP:
2092               // fallthrough
2093             case NAT_PROTOCOL_UDP:
2094               is_hairpinning = nat44_ei_hairpinning (
2095                 vm, node, nm, thread_index, b0, ip0, udp0, tcp0, proto0,
2096                 0 /* do_trace */, &required_thread_index);
2097               break;
2098             case NAT_PROTOCOL_ICMP:
2099               is_hairpinning = (0 == nat44_ei_icmp_hairpinning (
2100                                        nm, b0, thread_index, ip0, icmp0,
2101                                        &required_thread_index));
2102               break;
2103             case NAT_PROTOCOL_OTHER:
2104               // this should never happen
2105               next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2106               break;
2107             }
2108
2109           if (thread_index != required_thread_index)
2110             {
2111               // but we already did a handoff ...
2112               next0 = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP;
2113             }
2114
2115           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) &&
2116                              (b0->flags & VLIB_BUFFER_IS_TRACED)))
2117             {
2118               nat44_ei_in2out_trace_t *t =
2119                 vlib_add_trace (vm, node, b0, sizeof (*t));
2120               t->sw_if_index = sw_if_index0;
2121               t->next_index = next0;
2122               t->is_hairpinning = is_hairpinning;
2123             }
2124
2125           if (next0 != NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP)
2126             {
2127               vlib_increment_simple_counter (
2128                 &nm->counters.fastpath.in2out.other, sw_if_index0,
2129                 vm->thread_index, 1);
2130             }
2131
2132           /* verify speculative enqueue, maybe switch current next frame */
2133           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2134                                            n_left_to_next, bi0, next0);
2135         }
2136
2137       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2138     }
2139
2140   return frame->n_vectors;
2141 }
2142
2143 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_finish_ip4_lookup_node)
2144 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2145 {
2146   return nat44_ei_in2out_hairpinning_finish_inline (vm, node, frame);
2147 }
2148
2149 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_finish_ip4_lookup_node) = {
2150   .name = "nat44-ei-in2out-hairpinning-finish-ip4-lookup",
2151   .vector_size = sizeof (u32),
2152   .format_trace = format_nat44_ei_in2out_fast_trace,
2153   .type = VLIB_NODE_TYPE_INTERNAL,
2154
2155   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2156   .error_strings = nat44_ei_in2out_error_strings,
2157
2158   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2159
2160   .n_next_nodes = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2161
2162   /* edit / add dispositions here */
2163   .next_nodes = {
2164     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2165     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "ip4-lookup",
2166   },
2167 };
2168
2169 VLIB_NODE_FN (nat44_ei_in2out_hairpinning_finish_interface_output_node)
2170 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2171 {
2172   return nat44_ei_in2out_hairpinning_finish_inline (vm, node, frame);
2173 }
2174
2175 VLIB_REGISTER_NODE (nat44_ei_in2out_hairpinning_finish_interface_output_node) = {
2176   .name = "nat44-ei-in2out-hairpinning-finish-interface-output",
2177   .vector_size = sizeof (u32),
2178   .format_trace = format_nat44_ei_in2out_fast_trace,
2179   .type = VLIB_NODE_TYPE_INTERNAL,
2180
2181   .n_errors = ARRAY_LEN(nat44_ei_in2out_error_strings),
2182   .error_strings = nat44_ei_in2out_error_strings,
2183
2184   .runtime_data_bytes = sizeof (nat44_ei_runtime_t),
2185
2186   .n_next_nodes = NAT44_EI_IN2OUT_HAIRPINNING_FINISH_N_NEXT,
2187
2188   /* edit / add dispositions here */
2189   .next_nodes = {
2190     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_DROP] = "error-drop",
2191     [NAT44_EI_IN2OUT_HAIRPINNING_FINISH_NEXT_LOOKUP] = "interface-output",
2192   },
2193 };
2194
2195 /*
2196  * fd.io coding-style-patch-verification: ON
2197  *
2198  * Local Variables:
2199  * eval: (c-set-style "gnu")
2200  * End:
2201  */