src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28 #include <nat/nat_inlines.h>
  29
  30 #include <vppinfra/hash.h>
  31 #include <vppinfra/error.h>
  32 #include <vppinfra/elog.h>
  33
  34 typedef struct {
  35   u32 sw_if_index;
  36   u32 next_index;
  37   u32 session_index;
  38   u32 is_slow_path;
  39 } snat_in2out_trace_t;
  40
  41 typedef struct {
  42   u32 next_worker_index;
  43   u8 do_handoff;
  44 } snat_in2out_worker_handoff_trace_t;
  45
  46 /* packet trace format function */
  47 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  48 {
  49   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  50   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  51   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  52   char * tag;
  53
  54   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  55
  56   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  57               t->sw_if_index, t->next_index, t->session_index);
  58
  59   return s;
  60 }
  61
  62 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  63 {
  64   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  65   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  66   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  67
  68   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  69               t->sw_if_index, t->next_index);
  70
  71   return s;
  72 }
  73
  74 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  75 {
  76   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  77   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  78   snat_in2out_worker_handoff_trace_t * t =
  79     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  80   char * m;
  81
  82   m = t->do_handoff ? "next worker" : "same worker";
  83   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  84
  85   return s;
  86 }
  87
  88 typedef struct {
  89   u32 sw_if_index;
  90   u32 next_index;
  91   u8 cached;
  92 } nat44_in2out_reass_trace_t;
  93
  94 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  95 {
  96   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  97   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  98   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  99
 100   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 101               t->sw_if_index, t->next_index,
 102               t->cached ? "cached" : "translated");
 103
 104   return s;
 105 }
 106
 107 vlib_node_registration_t snat_in2out_node;
 108 vlib_node_registration_t snat_in2out_slowpath_node;
 109 vlib_node_registration_t snat_in2out_fast_node;
 110 vlib_node_registration_t snat_in2out_worker_handoff_node;
 111 vlib_node_registration_t snat_det_in2out_node;
 112 vlib_node_registration_t snat_in2out_output_node;
 113 vlib_node_registration_t snat_in2out_output_slowpath_node;
 114 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 115 vlib_node_registration_t snat_hairpin_dst_node;
 116 vlib_node_registration_t snat_hairpin_src_node;
 117 vlib_node_registration_t nat44_hairpinning_node;
 118 vlib_node_registration_t nat44_in2out_reass_node;
 119 vlib_node_registration_t nat44_ed_in2out_node;
 120 vlib_node_registration_t nat44_ed_in2out_slowpath_node;
 121 vlib_node_registration_t nat44_ed_in2out_output_node;
 122 vlib_node_registration_t nat44_ed_in2out_output_slowpath_node;
 123 vlib_node_registration_t nat44_ed_hairpin_dst_node;
 124 vlib_node_registration_t nat44_ed_hairpin_src_node;
 125 vlib_node_registration_t nat44_ed_hairpinning_node;
 126
 127 #define foreach_snat_in2out_error                       \
 128 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 129 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 130 _(OUT_OF_PORTS, "Out of ports")                         \
 131 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 132 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 133 _(NO_TRANSLATION, "No translation")                     \
 134 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 135 _(DROP_FRAGMENT, "Drop fragment")                       \
 136 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 137 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")\
 138 _(FQ_CONGESTED, "Handoff frame queue congested")
 139
 140 typedef enum {
 141 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 142   foreach_snat_in2out_error
 143 #undef _
 144   SNAT_IN2OUT_N_ERROR,
 145 } snat_in2out_error_t;
 146
 147 static char * snat_in2out_error_strings[] = {
 148 #define _(sym,string) string,
 149   foreach_snat_in2out_error
 150 #undef _
 151 };
 152
 153 typedef enum {
 154   SNAT_IN2OUT_NEXT_LOOKUP,
 155   SNAT_IN2OUT_NEXT_DROP,
 156   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 157   SNAT_IN2OUT_NEXT_SLOW_PATH,
 158   SNAT_IN2OUT_NEXT_REASS,
 159   SNAT_IN2OUT_N_NEXT,
 160 } snat_in2out_next_t;
 161
 162 typedef enum {
 163   SNAT_HAIRPIN_SRC_NEXT_DROP,
 164   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 165   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 166   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 167   SNAT_HAIRPIN_SRC_N_NEXT,
 168 } snat_hairpin_next_t;
 169
 170 /**
 171  * @brief Check if packet should be translated
 172  *
 173  * Packets aimed at outside interface and external address with active session
 174  * should be translated.
 175  *
 176  * @param sm            NAT main
 177  * @param rt            NAT runtime data
 178  * @param sw_if_index0  index of the inside interface
 179  * @param ip0           IPv4 header
 180  * @param proto0        NAT protocol
 181  * @param rx_fib_index0 RX FIB index
 182  *
 183  * @returns 0 if packet should be translated otherwise 1
 184  */
 185 static inline int
 186 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 187                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 188                          u32 rx_fib_index0)
 189 {
 190   if (sm->out2in_dpo)
 191     return 0;
 192
 193   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 194   nat_outside_fib_t *outside_fib;
 195   fib_prefix_t pfx = {
 196     .fp_proto = FIB_PROTOCOL_IP4,
 197     .fp_len = 32,
 198     .fp_addr = {
 199         .ip4.as_u32 = ip0->dst_address.as_u32,
 200     },
 201   };
 202
 203   /* Don't NAT packet aimed at the intfc address */
 204   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 205                                       ip0->dst_address.as_u32)))
 206     return 1;
 207
 208   fei = fib_table_lookup (rx_fib_index0, &pfx);
 209   if (FIB_NODE_INDEX_INVALID != fei)
 210     {
 211       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 212       if (sw_if_index == ~0)
 213         {
 214           vec_foreach (outside_fib, sm->outside_fibs)
 215             {
 216               fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 217               if (FIB_NODE_INDEX_INVALID != fei)
 218                 {
 219                   sw_if_index = fib_entry_get_resolving_interface (fei);
 220                   if (sw_if_index != ~0)
 221                     break;
 222                 }
 223             }
 224         }
 225       if (sw_if_index == ~0)
 226         return 1;
 227
 228       snat_interface_t *i;
 229       pool_foreach (i, sm->interfaces,
 230       ({
 231         /* NAT packet aimed at outside interface */
 232         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 233           return 0;
 234       }));
 235     }
 236
 237   return 1;
 238 }
 239
 240 static inline int
 241 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 242                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 243                     u32 rx_fib_index0, u32 thread_index)
 244 {
 245   udp_header_t * udp0 = ip4_next_header (ip0);
 246   snat_session_key_t key0, sm0;
 247   clib_bihash_kv_8_8_t kv0, value0;
 248
 249   key0.addr = ip0->dst_address;
 250   key0.port = udp0->dst_port;
 251   key0.protocol = proto0;
 252   key0.fib_index = sm->outside_fib_index;
 253   kv0.key = key0.as_u64;
 254
 255   /* NAT packet aimed at external address if */
 256   /* has active sessions */
 257   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 258                               &value0))
 259     {
 260       /* or is static mappings */
 261       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 262         return 0;
 263     }
 264   else
 265     return 0;
 266
 267   if (sm->forwarding_enabled)
 268     return 1;
 269
 270   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 271                                  rx_fib_index0);
 272 }
 273
 274 static inline int
 275 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 276                                   u32 proto0, u16 src_port, u16 dst_port,
 277                                   u32 thread_index, u32 sw_if_index)
 278 {
 279   snat_session_key_t key0;
 280   clib_bihash_kv_8_8_t kv0, value0;
 281   snat_interface_t *i;
 282
 283   /* src NAT check */
 284   key0.addr = ip0->src_address;
 285   key0.port = src_port;
 286   key0.protocol = proto0;
 287   key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 288   kv0.key = key0.as_u64;
 289
 290   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 291                                &value0))
 292     return 1;
 293
 294   /* dst NAT check */
 295   key0.addr = ip0->dst_address;
 296   key0.port = dst_port;
 297   key0.protocol = proto0;
 298   kv0.key = key0.as_u64;
 299   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 300                                &value0))
 301   {
 302     /* hairpinning */
 303     pool_foreach (i, sm->output_feature_interfaces,
 304     ({
 305       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 306         return 0;
 307     }));
 308     return 1;
 309   }
 310
 311   return 0;
 312 }
 313
 314 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 315                       ip4_header_t * ip0,
 316                       u32 rx_fib_index0,
 317                       snat_session_key_t * key0,
 318                       snat_session_t ** sessionp,
 319                       vlib_node_runtime_t * node,
 320                       u32 next0,
 321                       u32 thread_index)
 322 {
 323   snat_user_t *u;
 324   snat_session_t *s;
 325   clib_bihash_kv_8_8_t kv0;
 326   snat_session_key_t key1;
 327   u32 address_index = ~0;
 328   udp_header_t * udp0 = ip4_next_header (ip0);
 329   u8 is_sm = 0;
 330   nat_outside_fib_t *outside_fib;
 331   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 332   fib_prefix_t pfx = {
 333     .fp_proto = FIB_PROTOCOL_IP4,
 334     .fp_len = 32,
 335     .fp_addr = {
 336         .ip4.as_u32 = ip0->dst_address.as_u32,
 337     },
 338   };
 339
 340   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 341     {
 342       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 343       nat_ipfix_logging_max_sessions(sm->max_translations);
 344       nat_log_notice ("maximum sessions exceeded");
 345       return SNAT_IN2OUT_NEXT_DROP;
 346     }
 347
 348   key1.protocol = key0->protocol;
 349
 350   /* First try to match static mapping by local address and port */
 351   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
 352     {
 353       /* Try to create dynamic translation */
 354       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 355                                                thread_index, &key1,
 356                                                &address_index,
 357                                                sm->port_per_thread,
 358                                                sm->per_thread_data[thread_index].snat_thread_index))
 359         {
 360           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 361           return SNAT_IN2OUT_NEXT_DROP;
 362         }
 363     }
 364   else
 365     is_sm = 1;
 366
 367   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 368                               thread_index);
 369   if (!u)
 370     {
 371       nat_log_warn ("create NAT user failed");
 372       return SNAT_IN2OUT_NEXT_DROP;
 373     }
 374
 375   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 376   if (!s)
 377     {
 378       nat44_delete_user_with_no_session (sm, u, thread_index);
 379       nat_log_warn ("create NAT session failed");
 380       return SNAT_IN2OUT_NEXT_DROP;
 381     }
 382
 383   if (is_sm)
 384     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 385   user_session_increment (sm, u, is_sm);
 386   s->outside_address_index = address_index;
 387   s->in2out = *key0;
 388   s->out2in = key1;
 389   s->out2in.protocol = key0->protocol;
 390   s->out2in.fib_index = sm->outside_fib_index;
 391   switch (vec_len (sm->outside_fibs))
 392     {
 393     case 0:
 394       s->out2in.fib_index = sm->outside_fib_index;
 395       break;
 396     case 1:
 397       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
 398       break;
 399     default:
 400       vec_foreach (outside_fib, sm->outside_fibs)
 401         {
 402           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
 403           if (FIB_NODE_INDEX_INVALID != fei)
 404             {
 405               if (fib_entry_get_resolving_interface (fei) != ~0)
 406                 {
 407                   s->out2in.fib_index = outside_fib->fib_index;
 408                   break;
 409                 }
 410             }
 411         }
 412       break;
 413     }
 414   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 415   s->ext_host_port = udp0->dst_port;
 416   *sessionp = s;
 417
 418   /* Add to translation hashes */
 419   kv0.key = s->in2out.as_u64;
 420   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 421   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 422                                1 /* is_add */))
 423       nat_log_notice ("in2out key add failed");
 424
 425   kv0.key = s->out2in.as_u64;
 426   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 427
 428   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 429                                1 /* is_add */))
 430       nat_log_notice ("out2in key add failed");
 431
 432   /* log NAT event */
 433   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 434                                       s->out2in.addr.as_u32,
 435                                       s->in2out.protocol,
 436                                       s->in2out.port,
 437                                       s->out2in.port,
 438                                       s->in2out.fib_index);
 439   return next0;
 440 }
 441
 442 static_always_inline
 443 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 444                                  snat_session_key_t *p_key0)
 445 {
 446   icmp46_header_t *icmp0;
 447   snat_session_key_t key0;
 448   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 449   ip4_header_t *inner_ip0 = 0;
 450   void *l4_header = 0;
 451   icmp46_header_t *inner_icmp0;
 452
 453   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 454   echo0 = (icmp_echo_header_t *)(icmp0+1);
 455
 456   if (!icmp_is_error_message (icmp0))
 457     {
 458       key0.protocol = SNAT_PROTOCOL_ICMP;
 459       key0.addr = ip0->src_address;
 460       key0.port = echo0->identifier;
 461     }
 462   else
 463     {
 464       inner_ip0 = (ip4_header_t *)(echo0+1);
 465       l4_header = ip4_next_header (inner_ip0);
 466       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 467       key0.addr = inner_ip0->dst_address;
 468       switch (key0.protocol)
 469         {
 470         case SNAT_PROTOCOL_ICMP:
 471           inner_icmp0 = (icmp46_header_t*)l4_header;
 472           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 473           key0.port = inner_echo0->identifier;
 474           break;
 475         case SNAT_PROTOCOL_UDP:
 476         case SNAT_PROTOCOL_TCP:
 477           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 478           break;
 479         default:
 480           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 481         }
 482     }
 483   *p_key0 = key0;
 484   return -1; /* success */
 485 }
 486
 487 /**
 488  * Get address and port values to be used for ICMP packet translation
 489  * and create session if needed
 490  *
 491  * @param[in,out] sm             NAT main
 492  * @param[in,out] node           NAT node runtime
 493  * @param[in] thread_index       thread index
 494  * @param[in,out] b0             buffer containing packet to be translated
 495  * @param[out] p_proto           protocol used for matching
 496  * @param[out] p_value           address and port after NAT translation
 497  * @param[out] p_dont_translate  if packet should not be translated
 498  * @param d                      optional parameter
 499  * @param e                      optional parameter
 500  */
 501 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 502                            u32 thread_index, vlib_buffer_t *b0,
 503                            ip4_header_t *ip0, u8 *p_proto,
 504                            snat_session_key_t *p_value,
 505                            u8 *p_dont_translate, void *d, void *e)
 506 {
 507   icmp46_header_t *icmp0;
 508   u32 sw_if_index0;
 509   u32 rx_fib_index0;
 510   snat_session_key_t key0;
 511   snat_session_t *s0 = 0;
 512   u8 dont_translate = 0;
 513   clib_bihash_kv_8_8_t kv0, value0;
 514   u32 next0 = ~0;
 515   int err;
 516
 517   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 518   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 519   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 520
 521   err = icmp_get_key (ip0, &key0);
 522   if (err != -1)
 523     {
 524       b0->error = node->errors[err];
 525       next0 = SNAT_IN2OUT_NEXT_DROP;
 526       goto out;
 527     }
 528   key0.fib_index = rx_fib_index0;
 529
 530   kv0.key = key0.as_u64;
 531
 532   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 533                               &value0))
 534     {
 535       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 536         {
 537           if (PREDICT_FALSE(nat_not_translate_output_feature(sm, ip0,
 538               key0.protocol, key0.port, key0.port, thread_index, sw_if_index0)))
 539             {
 540               dont_translate = 1;
 541               goto out;
 542             }
 543         }
 544       else
 545         {
 546           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 547               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 548             {
 549               dont_translate = 1;
 550               goto out;
 551             }
 552         }
 553
 554       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 555         {
 556           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 557           next0 = SNAT_IN2OUT_NEXT_DROP;
 558           goto out;
 559         }
 560
 561       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 562                          &s0, node, next0, thread_index);
 563
 564       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 565         goto out;
 566     }
 567   else
 568     {
 569       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 570                         icmp0->type != ICMP4_echo_reply &&
 571                         !icmp_is_error_message (icmp0)))
 572         {
 573           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 574           next0 = SNAT_IN2OUT_NEXT_DROP;
 575           goto out;
 576         }
 577
 578       s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 579                               value0.value);
 580     }
 581
 582 out:
 583   *p_proto = key0.protocol;
 584   if (s0)
 585     *p_value = s0->out2in;
 586   *p_dont_translate = dont_translate;
 587   if (d)
 588     *(snat_session_t**)d = s0;
 589   return next0;
 590 }
 591
 592 /**
 593  * Get address and port values to be used for ICMP packet translation
 594  *
 595  * @param[in] sm                 NAT main
 596  * @param[in,out] node           NAT node runtime
 597  * @param[in] thread_index       thread index
 598  * @param[in,out] b0             buffer containing packet to be translated
 599  * @param[out] p_proto           protocol used for matching
 600  * @param[out] p_value           address and port after NAT translation
 601  * @param[out] p_dont_translate  if packet should not be translated
 602  * @param d                      optional parameter
 603  * @param e                      optional parameter
 604  */
 605 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 606                            u32 thread_index, vlib_buffer_t *b0,
 607                            ip4_header_t *ip0, u8 *p_proto,
 608                            snat_session_key_t *p_value,
 609                            u8 *p_dont_translate, void *d, void *e)
 610 {
 611   icmp46_header_t *icmp0;
 612   u32 sw_if_index0;
 613   u32 rx_fib_index0;
 614   snat_session_key_t key0;
 615   snat_session_key_t sm0;
 616   u8 dont_translate = 0;
 617   u8 is_addr_only;
 618   u32 next0 = ~0;
 619   int err;
 620
 621   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 622   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 623   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 624
 625   err = icmp_get_key (ip0, &key0);
 626   if (err != -1)
 627     {
 628       b0->error = node->errors[err];
 629       next0 = SNAT_IN2OUT_NEXT_DROP;
 630       goto out2;
 631     }
 632   key0.fib_index = rx_fib_index0;
 633
 634   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
 635     {
 636       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 637           IP_PROTOCOL_ICMP, rx_fib_index0)))
 638         {
 639           dont_translate = 1;
 640           goto out;
 641         }
 642
 643       if (icmp_is_error_message (icmp0))
 644         {
 645           next0 = SNAT_IN2OUT_NEXT_DROP;
 646           goto out;
 647         }
 648
 649       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 650       next0 = SNAT_IN2OUT_NEXT_DROP;
 651       goto out;
 652     }
 653
 654   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 655                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 656                     !icmp_is_error_message (icmp0)))
 657     {
 658       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 659       next0 = SNAT_IN2OUT_NEXT_DROP;
 660       goto out;
 661     }
 662
 663 out:
 664   *p_value = sm0;
 665 out2:
 666   *p_proto = key0.protocol;
 667   *p_dont_translate = dont_translate;
 668   return next0;
 669 }
 670
 671 static inline u32 icmp_in2out (snat_main_t *sm,
 672                                vlib_buffer_t * b0,
 673                                ip4_header_t * ip0,
 674                                icmp46_header_t * icmp0,
 675                                u32 sw_if_index0,
 676                                u32 rx_fib_index0,
 677                                vlib_node_runtime_t * node,
 678                                u32 next0,
 679                                u32 thread_index,
 680                                void *d,
 681                                void *e)
 682 {
 683   snat_session_key_t sm0;
 684   u8 protocol;
 685   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 686   ip4_header_t *inner_ip0;
 687   void *l4_header = 0;
 688   icmp46_header_t *inner_icmp0;
 689   u8 dont_translate;
 690   u32 new_addr0, old_addr0;
 691   u16 old_id0, new_id0;
 692   ip_csum_t sum0;
 693   u16 checksum0;
 694   u32 next0_tmp;
 695
 696   echo0 = (icmp_echo_header_t *)(icmp0+1);
 697
 698   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 699                                        &protocol, &sm0, &dont_translate, d, e);
 700   if (next0_tmp != ~0)
 701     next0 = next0_tmp;
 702   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 703     goto out;
 704
 705   sum0 = ip_incremental_checksum (0, icmp0,
 706                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 707   checksum0 = ~ip_csum_fold (sum0);
 708   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 709     {
 710       next0 = SNAT_IN2OUT_NEXT_DROP;
 711       goto out;
 712     }
 713
 714   old_addr0 = ip0->src_address.as_u32;
 715   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 716   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 717     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 718
 719   sum0 = ip0->checksum;
 720   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 721                          src_address /* changed member */);
 722   ip0->checksum = ip_csum_fold (sum0);
 723
 724   if (icmp0->checksum == 0)
 725     icmp0->checksum = 0xffff;
 726
 727   if (!icmp_is_error_message (icmp0))
 728     {
 729       new_id0 = sm0.port;
 730       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 731         {
 732           old_id0 = echo0->identifier;
 733           new_id0 = sm0.port;
 734           echo0->identifier = new_id0;
 735
 736           sum0 = icmp0->checksum;
 737           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 738                                  identifier);
 739           icmp0->checksum = ip_csum_fold (sum0);
 740         }
 741     }
 742   else
 743     {
 744       inner_ip0 = (ip4_header_t *)(echo0+1);
 745       l4_header = ip4_next_header (inner_ip0);
 746
 747       if (!ip4_header_checksum_is_valid (inner_ip0))
 748         {
 749           next0 = SNAT_IN2OUT_NEXT_DROP;
 750           goto out;
 751         }
 752
 753       old_addr0 = inner_ip0->dst_address.as_u32;
 754       inner_ip0->dst_address = sm0.addr;
 755       new_addr0 = inner_ip0->dst_address.as_u32;
 756
 757       sum0 = icmp0->checksum;
 758       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 759                              dst_address /* changed member */);
 760       icmp0->checksum = ip_csum_fold (sum0);
 761
 762       switch (protocol)
 763         {
 764           case SNAT_PROTOCOL_ICMP:
 765             inner_icmp0 = (icmp46_header_t*)l4_header;
 766             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 767
 768             old_id0 = inner_echo0->identifier;
 769             new_id0 = sm0.port;
 770             inner_echo0->identifier = new_id0;
 771
 772             sum0 = icmp0->checksum;
 773             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 774                                    identifier);
 775             icmp0->checksum = ip_csum_fold (sum0);
 776             break;
 777           case SNAT_PROTOCOL_UDP:
 778           case SNAT_PROTOCOL_TCP:
 779             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 780             new_id0 = sm0.port;
 781             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 782
 783             sum0 = icmp0->checksum;
 784             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 785                                    dst_port);
 786             icmp0->checksum = ip_csum_fold (sum0);
 787             break;
 788           default:
 789             ASSERT(0);
 790         }
 791     }
 792
 793 out:
 794   return next0;
 795 }
 796
 797 /**
 798  * @brief Hairpinning
 799  *
 800  * Hairpinning allows two endpoints on the internal side of the NAT to
 801  * communicate even if they only use each other's external IP addresses
 802  * and ports.
 803  *
 804  * @param sm     NAT main.
 805  * @param b0     Vlib buffer.
 806  * @param ip0    IP header.
 807  * @param udp0   UDP header.
 808  * @param tcp0   TCP header.
 809  * @param proto0 NAT protocol.
 810  */
 811 static inline int
 812 snat_hairpinning (snat_main_t *sm,
 813                   vlib_buffer_t * b0,
 814                   ip4_header_t * ip0,
 815                   udp_header_t * udp0,
 816                   tcp_header_t * tcp0,
 817                   u32 proto0,
 818                   int is_ed)
 819 {
 820   snat_session_key_t key0, sm0;
 821   snat_session_t * s0;
 822   clib_bihash_kv_8_8_t kv0, value0;
 823   ip_csum_t sum0;
 824   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 825   u16 new_dst_port0, old_dst_port0;
 826   int rv;
 827
 828   key0.addr = ip0->dst_address;
 829   key0.port = udp0->dst_port;
 830   key0.protocol = proto0;
 831   key0.fib_index = sm->outside_fib_index;
 832   kv0.key = key0.as_u64;
 833
 834   /* Check if destination is static mappings */
 835   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 836     {
 837       new_dst_addr0 = sm0.addr.as_u32;
 838       new_dst_port0 = sm0.port;
 839       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 840     }
 841   /* or active session */
 842   else
 843     {
 844       if (sm->num_workers > 1)
 845         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 846       else
 847         ti = sm->num_workers;
 848
 849       if (is_ed)
 850         {
 851           clib_bihash_kv_16_8_t ed_kv, ed_value;
 852           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 853                       ip0->protocol, sm->outside_fib_index, udp0->dst_port,
 854                       udp0->src_port);
 855           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 856                                         &ed_kv, &ed_value);
 857           si = ed_value.value;
 858         }
 859       else
 860         {
 861           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 862                                        &value0);
 863           si = value0.value;
 864         }
 865       if (rv)
 866         return 0;
 867
 868       s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 869       new_dst_addr0 = s0->in2out.addr.as_u32;
 870       new_dst_port0 = s0->in2out.port;
 871       vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 872     }
 873
 874   /* Destination is behind the same NAT, use internal address and port */
 875   if (new_dst_addr0)
 876     {
 877       old_dst_addr0 = ip0->dst_address.as_u32;
 878       ip0->dst_address.as_u32 = new_dst_addr0;
 879       sum0 = ip0->checksum;
 880       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 881                              ip4_header_t, dst_address);
 882       ip0->checksum = ip_csum_fold (sum0);
 883
 884       old_dst_port0 = tcp0->dst;
 885       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 886         {
 887           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 888             {
 889               tcp0->dst = new_dst_port0;
 890               sum0 = tcp0->checksum;
 891               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 892                                      ip4_header_t, dst_address);
 893               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 894                                      ip4_header_t /* cheat */, length);
 895               tcp0->checksum = ip_csum_fold(sum0);
 896             }
 897           else
 898             {
 899               udp0->dst_port = new_dst_port0;
 900               udp0->checksum = 0;
 901             }
 902         }
 903       else
 904         {
 905           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 906             {
 907               sum0 = tcp0->checksum;
 908               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 909                                      ip4_header_t, dst_address);
 910               tcp0->checksum = ip_csum_fold(sum0);
 911             }
 912         }
 913       return 1;
 914     }
 915   return 0;
 916 }
 917
 918 static inline void
 919 snat_icmp_hairpinning (snat_main_t *sm,
 920                        vlib_buffer_t * b0,
 921                        ip4_header_t * ip0,
 922                        icmp46_header_t * icmp0,
 923                        int is_ed)
 924 {
 925   snat_session_key_t key0, sm0;
 926   clib_bihash_kv_8_8_t kv0, value0;
 927   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 928   ip_csum_t sum0;
 929   snat_session_t *s0;
 930   int rv;
 931
 932   if (!icmp_is_error_message (icmp0))
 933     {
 934       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 935       u16 icmp_id0 = echo0->identifier;
 936       key0.addr = ip0->dst_address;
 937       key0.port = icmp_id0;
 938       key0.protocol = SNAT_PROTOCOL_ICMP;
 939       key0.fib_index = sm->outside_fib_index;
 940       kv0.key = key0.as_u64;
 941
 942       if (sm->num_workers > 1)
 943         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 944       else
 945         ti = sm->num_workers;
 946
 947       /* Check if destination is in active sessions */
 948       if (is_ed)
 949         {
 950           clib_bihash_kv_16_8_t ed_kv, ed_value;
 951           make_ed_kv (&ed_kv, &ip0->dst_address, &ip0->src_address,
 952                       IP_PROTOCOL_ICMP, sm->outside_fib_index, icmp_id0, 0);
 953           rv = clib_bihash_search_16_8 (&sm->per_thread_data[ti].out2in_ed,
 954                                         &ed_kv, &ed_value);
 955           si = ed_value.value;
 956         }
 957       else
 958         {
 959           rv = clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 960                                        &value0);
 961           si = value0.value;
 962         }
 963       if (rv)
 964         {
 965           /* or static mappings */
 966           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 967             {
 968               new_dst_addr0 = sm0.addr.as_u32;
 969               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 970             }
 971         }
 972       else
 973         {
 974           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 975           new_dst_addr0 = s0->in2out.addr.as_u32;
 976           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 977           echo0->identifier = s0->in2out.port;
 978           sum0 = icmp0->checksum;
 979           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
 980                                  icmp_echo_header_t, identifier);
 981           icmp0->checksum = ip_csum_fold (sum0);
 982         }
 983
 984       /* Destination is behind the same NAT, use internal address and port */
 985       if (new_dst_addr0)
 986         {
 987           old_dst_addr0 = ip0->dst_address.as_u32;
 988           ip0->dst_address.as_u32 = new_dst_addr0;
 989           sum0 = ip0->checksum;
 990           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 991                                  ip4_header_t, dst_address);
 992           ip0->checksum = ip_csum_fold (sum0);
 993         }
 994     }
 995
 996 }
 997
 998 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
 999                                          vlib_buffer_t * b0,
1000                                          ip4_header_t * ip0,
1001                                          icmp46_header_t * icmp0,
1002                                          u32 sw_if_index0,
1003                                          u32 rx_fib_index0,
1004                                          vlib_node_runtime_t * node,
1005                                          u32 next0,
1006                                          f64 now,
1007                                          u32 thread_index,
1008                                          snat_session_t ** p_s0)
1009 {
1010   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1011                       next0, thread_index, p_s0, 0);
1012   snat_session_t * s0 = *p_s0;
1013   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1014     {
1015       /* Hairpinning */
1016       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1017         snat_icmp_hairpinning(sm, b0, ip0, icmp0, sm->endpoint_dependent);
1018       /* Accounting */
1019       nat44_session_update_counters (s0, now,
1020                                      vlib_buffer_length_in_chain (sm->vlib_main, b0));
1021       /* Per-user LRU list maintenance */
1022       nat44_session_update_lru (sm, s0, thread_index);
1023     }
1024   return next0;
1025 }
1026
1027 static inline void
1028 nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
1029                                   vlib_buffer_t * b,
1030                                   ip4_header_t * ip)
1031 {
1032   clib_bihash_kv_8_8_t kv, value;
1033   snat_static_mapping_t *m;
1034   u32 old_addr, new_addr;
1035   ip_csum_t sum;
1036
1037   make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
1038   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1039     return;
1040
1041   m = pool_elt_at_index (sm->static_mappings, value.value);
1042
1043   old_addr = ip->dst_address.as_u32;
1044   new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1045   sum = ip->checksum;
1046   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1047   ip->checksum = ip_csum_fold (sum);
1048
1049   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1050     vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1051 }
1052
1053 static int
1054 nat_in2out_sm_unknown_proto (snat_main_t *sm,
1055                              vlib_buffer_t * b,
1056                              ip4_header_t * ip,
1057                              u32 rx_fib_index)
1058 {
1059   clib_bihash_kv_8_8_t kv, value;
1060   snat_static_mapping_t *m;
1061   snat_session_key_t m_key;
1062   u32 old_addr, new_addr;
1063   ip_csum_t sum;
1064
1065   m_key.addr = ip->src_address;
1066   m_key.port = 0;
1067   m_key.protocol = 0;
1068   m_key.fib_index = rx_fib_index;
1069   kv.key = m_key.as_u64;
1070   if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1071     return 1;
1072
1073   m = pool_elt_at_index (sm->static_mappings, value.value);
1074
1075   old_addr = ip->src_address.as_u32;
1076   new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1077   sum = ip->checksum;
1078   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1079   ip->checksum = ip_csum_fold (sum);
1080
1081
1082   /* Hairpinning */
1083   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1084     {
1085       vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1086       nat_hairpinning_sm_unknown_proto (sm, b, ip);
1087     }
1088
1089   return 0;
1090 }
1091
1092 static inline uword
1093 snat_in2out_node_fn_inline (vlib_main_t * vm,
1094                             vlib_node_runtime_t * node,
1095                             vlib_frame_t * frame, int is_slow_path,
1096                             int is_output_feature)
1097 {
1098   u32 n_left_from, * from, * to_next;
1099   snat_in2out_next_t next_index;
1100   u32 pkts_processed = 0;
1101   snat_main_t * sm = &snat_main;
1102   f64 now = vlib_time_now (vm);
1103   u32 stats_node_index;
1104   u32 thread_index = vm->thread_index;
1105
1106   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1107     snat_in2out_node.index;
1108
1109   from = vlib_frame_vector_args (frame);
1110   n_left_from = frame->n_vectors;
1111   next_index = node->cached_next_index;
1112
1113   while (n_left_from > 0)
1114     {
1115       u32 n_left_to_next;
1116
1117       vlib_get_next_frame (vm, node, next_index,
1118                            to_next, n_left_to_next);
1119
1120       while (n_left_from >= 4 && n_left_to_next >= 2)
1121         {
1122           u32 bi0, bi1;
1123           vlib_buffer_t * b0, * b1;
1124           u32 next0, next1;
1125           u32 sw_if_index0, sw_if_index1;
1126           ip4_header_t * ip0, * ip1;
1127           ip_csum_t sum0, sum1;
1128           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1129           u16 old_port0, new_port0, old_port1, new_port1;
1130           udp_header_t * udp0, * udp1;
1131           tcp_header_t * tcp0, * tcp1;
1132           icmp46_header_t * icmp0, * icmp1;
1133           snat_session_key_t key0, key1;
1134           u32 rx_fib_index0, rx_fib_index1;
1135           u32 proto0, proto1;
1136           snat_session_t * s0 = 0, * s1 = 0;
1137           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1138           u32 iph_offset0 = 0, iph_offset1 = 0;
1139
1140           /* Prefetch next iteration. */
1141           {
1142             vlib_buffer_t * p2, * p3;
1143
1144             p2 = vlib_get_buffer (vm, from[2]);
1145             p3 = vlib_get_buffer (vm, from[3]);
1146
1147             vlib_prefetch_buffer_header (p2, LOAD);
1148             vlib_prefetch_buffer_header (p3, LOAD);
1149
1150             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1151             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1152           }
1153
1154           /* speculatively enqueue b0 and b1 to the current next frame */
1155           to_next[0] = bi0 = from[0];
1156           to_next[1] = bi1 = from[1];
1157           from += 2;
1158           to_next += 2;
1159           n_left_from -= 2;
1160           n_left_to_next -= 2;
1161
1162           b0 = vlib_get_buffer (vm, bi0);
1163           b1 = vlib_get_buffer (vm, bi1);
1164
1165           if (is_output_feature)
1166             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1167
1168           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1169                  iph_offset0);
1170
1171           udp0 = ip4_next_header (ip0);
1172           tcp0 = (tcp_header_t *) udp0;
1173           icmp0 = (icmp46_header_t *) udp0;
1174
1175           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1176           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1177                                    sw_if_index0);
1178
1179           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1180
1181           if (PREDICT_FALSE(ip0->ttl == 1))
1182             {
1183               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1184               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1185                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1186                                            0);
1187               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1188               goto trace00;
1189             }
1190
1191           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1192
1193           /* Next configured feature, probably ip4-lookup */
1194           if (is_slow_path)
1195             {
1196               if (PREDICT_FALSE (proto0 == ~0))
1197                 {
1198                   if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1199                     {
1200                       next0 = SNAT_IN2OUT_NEXT_DROP;
1201                       b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1202                     }
1203                   goto trace00;
1204                 }
1205
1206               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1207                 {
1208                   next0 = icmp_in2out_slow_path
1209                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1210                      node, next0, now, thread_index, &s0);
1211                   goto trace00;
1212                 }
1213             }
1214           else
1215             {
1216               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1217                 {
1218                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1219                   goto trace00;
1220                 }
1221
1222               if (ip4_is_fragment (ip0))
1223                 {
1224                   next0 = SNAT_IN2OUT_NEXT_REASS;
1225                   goto trace00;
1226                 }
1227             }
1228
1229           key0.addr = ip0->src_address;
1230           key0.port = udp0->src_port;
1231           key0.protocol = proto0;
1232           key0.fib_index = rx_fib_index0;
1233
1234           kv0.key = key0.as_u64;
1235
1236           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1237               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1238             {
1239               if (is_slow_path)
1240                 {
1241                   if (is_output_feature)
1242                     {
1243                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1244                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1245                         goto trace00;
1246                     }
1247                   else
1248                     {
1249                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1250                           ip0, proto0, rx_fib_index0, thread_index)))
1251                         goto trace00;
1252                     }
1253
1254                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1255                                      &s0, node, next0, thread_index);
1256                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1257                     goto trace00;
1258                 }
1259               else
1260                 {
1261                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1262                   goto trace00;
1263                 }
1264             }
1265           else
1266             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1267                                     value0.value);
1268
1269           b0->flags |= VNET_BUFFER_F_IS_NATED;
1270
1271           old_addr0 = ip0->src_address.as_u32;
1272           ip0->src_address = s0->out2in.addr;
1273           new_addr0 = ip0->src_address.as_u32;
1274           if (!is_output_feature)
1275             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1276
1277           sum0 = ip0->checksum;
1278           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1279                                  ip4_header_t,
1280                                  src_address /* changed member */);
1281           ip0->checksum = ip_csum_fold (sum0);
1282
1283           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1284             {
1285               old_port0 = tcp0->src_port;
1286               tcp0->src_port = s0->out2in.port;
1287               new_port0 = tcp0->src_port;
1288
1289               sum0 = tcp0->checksum;
1290               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1291                                      ip4_header_t,
1292                                      dst_address /* changed member */);
1293               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1294                                      ip4_header_t /* cheat */,
1295                                      length /* changed member */);
1296               tcp0->checksum = ip_csum_fold(sum0);
1297             }
1298           else
1299             {
1300               old_port0 = udp0->src_port;
1301               udp0->src_port = s0->out2in.port;
1302               udp0->checksum = 0;
1303             }
1304
1305           /* Accounting */
1306           nat44_session_update_counters (s0, now,
1307                                          vlib_buffer_length_in_chain (vm, b0));
1308           /* Per-user LRU list maintenance */
1309           nat44_session_update_lru (sm, s0, thread_index);
1310         trace00:
1311
1312           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1313                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1314             {
1315               snat_in2out_trace_t *t =
1316                  vlib_add_trace (vm, node, b0, sizeof (*t));
1317               t->is_slow_path = is_slow_path;
1318               t->sw_if_index = sw_if_index0;
1319               t->next_index = next0;
1320                   t->session_index = ~0;
1321               if (s0)
1322                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1323             }
1324
1325           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1326
1327           if (is_output_feature)
1328             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1329
1330           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1331                  iph_offset1);
1332
1333           udp1 = ip4_next_header (ip1);
1334           tcp1 = (tcp_header_t *) udp1;
1335           icmp1 = (icmp46_header_t *) udp1;
1336
1337           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1338           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1339                                    sw_if_index1);
1340
1341           if (PREDICT_FALSE(ip1->ttl == 1))
1342             {
1343               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1344               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1345                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1346                                            0);
1347               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1348               goto trace01;
1349             }
1350
1351           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1352
1353           /* Next configured feature, probably ip4-lookup */
1354           if (is_slow_path)
1355             {
1356               if (PREDICT_FALSE (proto1 == ~0))
1357                 {
1358                   if (nat_in2out_sm_unknown_proto (sm, b1, ip1, rx_fib_index1))
1359                     {
1360                       next1 = SNAT_IN2OUT_NEXT_DROP;
1361                       b1->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1362                     }
1363                   goto trace01;
1364                 }
1365
1366               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1367                 {
1368                   next1 = icmp_in2out_slow_path
1369                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1370                      next1, now, thread_index, &s1);
1371                   goto trace01;
1372                 }
1373             }
1374           else
1375             {
1376               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1377                 {
1378                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1379                   goto trace01;
1380                 }
1381
1382               if (ip4_is_fragment (ip1))
1383                 {
1384                   next1 = SNAT_IN2OUT_NEXT_REASS;
1385                   goto trace01;
1386                 }
1387             }
1388
1389           key1.addr = ip1->src_address;
1390           key1.port = udp1->src_port;
1391           key1.protocol = proto1;
1392           key1.fib_index = rx_fib_index1;
1393
1394           kv1.key = key1.as_u64;
1395
1396             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1397                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1398             {
1399               if (is_slow_path)
1400                 {
1401                   if (is_output_feature)
1402                     {
1403                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1404                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1405                         goto trace01;
1406                     }
1407                   else
1408                     {
1409                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1410                           ip1, proto1, rx_fib_index1, thread_index)))
1411                         goto trace01;
1412                     }
1413
1414                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1415                                      &s1, node, next1, thread_index);
1416                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1417                     goto trace01;
1418                 }
1419               else
1420                 {
1421                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1422                   goto trace01;
1423                 }
1424             }
1425           else
1426             s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1427                                     value1.value);
1428
1429           b1->flags |= VNET_BUFFER_F_IS_NATED;
1430
1431           old_addr1 = ip1->src_address.as_u32;
1432           ip1->src_address = s1->out2in.addr;
1433           new_addr1 = ip1->src_address.as_u32;
1434           if (!is_output_feature)
1435             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1436
1437           sum1 = ip1->checksum;
1438           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1439                                  ip4_header_t,
1440                                  src_address /* changed member */);
1441           ip1->checksum = ip_csum_fold (sum1);
1442
1443           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1444             {
1445               old_port1 = tcp1->src_port;
1446               tcp1->src_port = s1->out2in.port;
1447               new_port1 = tcp1->src_port;
1448
1449               sum1 = tcp1->checksum;
1450               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1451                                      ip4_header_t,
1452                                      dst_address /* changed member */);
1453               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1454                                      ip4_header_t /* cheat */,
1455                                      length /* changed member */);
1456               tcp1->checksum = ip_csum_fold(sum1);
1457             }
1458           else
1459             {
1460               old_port1 = udp1->src_port;
1461               udp1->src_port = s1->out2in.port;
1462               udp1->checksum = 0;
1463             }
1464
1465           /* Accounting */
1466           nat44_session_update_counters (s1, now,
1467                                          vlib_buffer_length_in_chain (vm, b1));
1468           /* Per-user LRU list maintenance */
1469           nat44_session_update_lru (sm, s1, thread_index);
1470         trace01:
1471
1472           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1473                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1474             {
1475               snat_in2out_trace_t *t =
1476                  vlib_add_trace (vm, node, b1, sizeof (*t));
1477               t->sw_if_index = sw_if_index1;
1478               t->next_index = next1;
1479               t->session_index = ~0;
1480               if (s1)
1481                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1482             }
1483
1484           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1485
1486           /* verify speculative enqueues, maybe switch current next frame */
1487           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1488                                            to_next, n_left_to_next,
1489                                            bi0, bi1, next0, next1);
1490         }
1491
1492       while (n_left_from > 0 && n_left_to_next > 0)
1493         {
1494           u32 bi0;
1495           vlib_buffer_t * b0;
1496           u32 next0;
1497           u32 sw_if_index0;
1498           ip4_header_t * ip0;
1499           ip_csum_t sum0;
1500           u32 new_addr0, old_addr0;
1501           u16 old_port0, new_port0;
1502           udp_header_t * udp0;
1503           tcp_header_t * tcp0;
1504           icmp46_header_t * icmp0;
1505           snat_session_key_t key0;
1506           u32 rx_fib_index0;
1507           u32 proto0;
1508           snat_session_t * s0 = 0;
1509           clib_bihash_kv_8_8_t kv0, value0;
1510           u32 iph_offset0 = 0;
1511
1512           /* speculatively enqueue b0 to the current next frame */
1513           bi0 = from[0];
1514           to_next[0] = bi0;
1515           from += 1;
1516           to_next += 1;
1517           n_left_from -= 1;
1518           n_left_to_next -= 1;
1519
1520           b0 = vlib_get_buffer (vm, bi0);
1521           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1522
1523           if (is_output_feature)
1524             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1525
1526           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1527                  iph_offset0);
1528
1529           udp0 = ip4_next_header (ip0);
1530           tcp0 = (tcp_header_t *) udp0;
1531           icmp0 = (icmp46_header_t *) udp0;
1532
1533           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1534           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1535                                    sw_if_index0);
1536
1537           if (PREDICT_FALSE(ip0->ttl == 1))
1538             {
1539               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1540               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1541                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1542                                            0);
1543               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1544               goto trace0;
1545             }
1546
1547           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1548
1549           /* Next configured feature, probably ip4-lookup */
1550           if (is_slow_path)
1551             {
1552               if (PREDICT_FALSE (proto0 == ~0))
1553                 {
1554                   if (nat_in2out_sm_unknown_proto (sm, b0, ip0, rx_fib_index0))
1555                     {
1556                       next0 = SNAT_IN2OUT_NEXT_DROP;
1557                       b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
1558                     }
1559                   goto trace0;
1560                 }
1561
1562               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1563                 {
1564                   next0 = icmp_in2out_slow_path
1565                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1566                      next0, now, thread_index, &s0);
1567                   goto trace0;
1568                 }
1569             }
1570           else
1571             {
1572               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1573                 {
1574                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1575                   goto trace0;
1576                 }
1577
1578               if (ip4_is_fragment (ip0))
1579                 {
1580                   next0 = SNAT_IN2OUT_NEXT_REASS;
1581                   goto trace0;
1582                 }
1583             }
1584
1585           key0.addr = ip0->src_address;
1586           key0.port = udp0->src_port;
1587           key0.protocol = proto0;
1588           key0.fib_index = rx_fib_index0;
1589
1590           kv0.key = key0.as_u64;
1591
1592           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
1593                                       &kv0, &value0))
1594             {
1595               if (is_slow_path)
1596                 {
1597                   if (is_output_feature)
1598                     {
1599                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1600                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1601                         goto trace0;
1602                     }
1603                   else
1604                     {
1605                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1606                           ip0, proto0, rx_fib_index0, thread_index)))
1607                         goto trace0;
1608                     }
1609
1610                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1611                                      &s0, node, next0, thread_index);
1612
1613                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1614                     goto trace0;
1615                 }
1616               else
1617                 {
1618                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1619                   goto trace0;
1620                 }
1621             }
1622           else
1623           s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
1624                                   value0.value);
1625
1626           b0->flags |= VNET_BUFFER_F_IS_NATED;
1627
1628           old_addr0 = ip0->src_address.as_u32;
1629           ip0->src_address = s0->out2in.addr;
1630           new_addr0 = ip0->src_address.as_u32;
1631           if (!is_output_feature)
1632             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1633
1634           sum0 = ip0->checksum;
1635           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1636                                  ip4_header_t,
1637                                  src_address /* changed member */);
1638           ip0->checksum = ip_csum_fold (sum0);
1639
1640           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1641             {
1642               old_port0 = tcp0->src_port;
1643               tcp0->src_port = s0->out2in.port;
1644               new_port0 = tcp0->src_port;
1645
1646               sum0 = tcp0->checksum;
1647               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1648                                      ip4_header_t,
1649                                      dst_address /* changed member */);
1650               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1651                                      ip4_header_t /* cheat */,
1652                                      length /* changed member */);
1653               tcp0->checksum = ip_csum_fold(sum0);
1654             }
1655           else
1656             {
1657               old_port0 = udp0->src_port;
1658               udp0->src_port = s0->out2in.port;
1659               udp0->checksum = 0;
1660             }
1661
1662           /* Accounting */
1663           nat44_session_update_counters (s0, now,
1664                                          vlib_buffer_length_in_chain (vm, b0));
1665           /* Per-user LRU list maintenance */
1666           nat44_session_update_lru (sm, s0, thread_index);
1667
1668         trace0:
1669           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1670                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1671             {
1672               snat_in2out_trace_t *t =
1673                  vlib_add_trace (vm, node, b0, sizeof (*t));
1674               t->is_slow_path = is_slow_path;
1675               t->sw_if_index = sw_if_index0;
1676               t->next_index = next0;
1677                   t->session_index = ~0;
1678               if (s0)
1679                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1680             }
1681
1682           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1683
1684           /* verify speculative enqueue, maybe switch current next frame */
1685           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1686                                            to_next, n_left_to_next,
1687                                            bi0, next0);
1688         }
1689
1690       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1691     }
1692
1693   vlib_node_increment_counter (vm, stats_node_index,
1694                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1695                                pkts_processed);
1696   return frame->n_vectors;
1697 }
1698
1699 static uword
1700 snat_in2out_fast_path_fn (vlib_main_t * vm,
1701                           vlib_node_runtime_t * node,
1702                           vlib_frame_t * frame)
1703 {
1704   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
1705 }
1706
1707 VLIB_REGISTER_NODE (snat_in2out_node) = {
1708   .function = snat_in2out_fast_path_fn,
1709   .name = "nat44-in2out",
1710   .vector_size = sizeof (u32),
1711   .format_trace = format_snat_in2out_trace,
1712   .type = VLIB_NODE_TYPE_INTERNAL,
1713
1714   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1715   .error_strings = snat_in2out_error_strings,
1716
1717   .runtime_data_bytes = sizeof (snat_runtime_t),
1718
1719   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1720
1721   /* edit / add dispositions here */
1722   .next_nodes = {
1723     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1724     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1725     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1726     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1727     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1728   },
1729 };
1730
1731 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
1732
1733 static uword
1734 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
1735                                  vlib_node_runtime_t * node,
1736                                  vlib_frame_t * frame)
1737 {
1738   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
1739 }
1740
1741 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
1742   .function = snat_in2out_output_fast_path_fn,
1743   .name = "nat44-in2out-output",
1744   .vector_size = sizeof (u32),
1745   .format_trace = format_snat_in2out_trace,
1746   .type = VLIB_NODE_TYPE_INTERNAL,
1747
1748   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1749   .error_strings = snat_in2out_error_strings,
1750
1751   .runtime_data_bytes = sizeof (snat_runtime_t),
1752
1753   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1754
1755   /* edit / add dispositions here */
1756   .next_nodes = {
1757     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1758     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1759     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1760     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1761     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1762   },
1763 };
1764
1765 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
1766                               snat_in2out_output_fast_path_fn);
1767
1768 static uword
1769 snat_in2out_slow_path_fn (vlib_main_t * vm,
1770                           vlib_node_runtime_t * node,
1771                           vlib_frame_t * frame)
1772 {
1773   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
1774 }
1775
1776 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
1777   .function = snat_in2out_slow_path_fn,
1778   .name = "nat44-in2out-slowpath",
1779   .vector_size = sizeof (u32),
1780   .format_trace = format_snat_in2out_trace,
1781   .type = VLIB_NODE_TYPE_INTERNAL,
1782
1783   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1784   .error_strings = snat_in2out_error_strings,
1785
1786   .runtime_data_bytes = sizeof (snat_runtime_t),
1787
1788   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1789
1790   /* edit / add dispositions here */
1791   .next_nodes = {
1792     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1793     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1794     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
1795     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1796     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1797   },
1798 };
1799
1800 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
1801                               snat_in2out_slow_path_fn);
1802
1803 static uword
1804 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
1805                                  vlib_node_runtime_t * node,
1806                                  vlib_frame_t * frame)
1807 {
1808   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
1809 }
1810
1811 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
1812   .function = snat_in2out_output_slow_path_fn,
1813   .name = "nat44-in2out-output-slowpath",
1814   .vector_size = sizeof (u32),
1815   .format_trace = format_snat_in2out_trace,
1816   .type = VLIB_NODE_TYPE_INTERNAL,
1817
1818   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1819   .error_strings = snat_in2out_error_strings,
1820
1821   .runtime_data_bytes = sizeof (snat_runtime_t),
1822
1823   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
1824
1825   /* edit / add dispositions here */
1826   .next_nodes = {
1827     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1828     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
1829     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
1830     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
1831     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
1832   },
1833 };
1834
1835 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
1836                               snat_in2out_output_slow_path_fn);
1837
1838 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
1839
1840 static inline uword
1841 nat44_hairpinning_fn_inline (vlib_main_t * vm,
1842                              vlib_node_runtime_t * node,
1843                              vlib_frame_t * frame,
1844                              int is_ed)
1845 {
1846   u32 n_left_from, * from, * to_next, stats_node_index;
1847   snat_in2out_next_t next_index;
1848   u32 pkts_processed = 0;
1849   snat_main_t * sm = &snat_main;
1850   vnet_feature_main_t *fm = &feature_main;
1851   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1852   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
1853
1854   stats_node_index = is_ed ? nat44_ed_hairpinning_node.index :
1855     nat44_hairpinning_node.index;
1856   from = vlib_frame_vector_args (frame);
1857   n_left_from = frame->n_vectors;
1858   next_index = node->cached_next_index;
1859
1860   while (n_left_from > 0)
1861     {
1862       u32 n_left_to_next;
1863
1864       vlib_get_next_frame (vm, node, next_index,
1865                            to_next, n_left_to_next);
1866
1867       while (n_left_from > 0 && n_left_to_next > 0)
1868         {
1869           u32 bi0;
1870           vlib_buffer_t * b0;
1871           u32 next0;
1872           ip4_header_t * ip0;
1873           u32 proto0;
1874           udp_header_t * udp0;
1875           tcp_header_t * tcp0;
1876
1877           /* speculatively enqueue b0 to the current next frame */
1878           bi0 = from[0];
1879           to_next[0] = bi0;
1880           from += 1;
1881           to_next += 1;
1882           n_left_from -= 1;
1883           n_left_to_next -= 1;
1884
1885           b0 = vlib_get_buffer (vm, bi0);
1886           ip0 = vlib_buffer_get_current (b0);
1887           udp0 = ip4_next_header (ip0);
1888           tcp0 = (tcp_header_t *) udp0;
1889
1890           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1891
1892           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
1893                                 &next0, 0);
1894
1895           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed))
1896             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1897
1898           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1899
1900           /* verify speculative enqueue, maybe switch current next frame */
1901           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1902                                            to_next, n_left_to_next,
1903                                            bi0, next0);
1904          }
1905
1906       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1907     }
1908
1909   vlib_node_increment_counter (vm, stats_node_index,
1910                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
1911                                pkts_processed);
1912   return frame->n_vectors;
1913 }
1914
1915 static uword
1916 nat44_hairpinning_fn (vlib_main_t * vm,
1917                       vlib_node_runtime_t * node,
1918                       vlib_frame_t * frame)
1919 {
1920   return nat44_hairpinning_fn_inline (vm, node, frame, 0);
1921 }
1922
1923 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
1924   .function = nat44_hairpinning_fn,
1925   .name = "nat44-hairpinning",
1926   .vector_size = sizeof (u32),
1927   .type = VLIB_NODE_TYPE_INTERNAL,
1928   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1929   .error_strings = snat_in2out_error_strings,
1930   .n_next_nodes = 2,
1931   .next_nodes = {
1932     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1933     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1934   },
1935 };
1936
1937 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
1938                               nat44_hairpinning_fn);
1939
1940 static uword
1941 nat44_ed_hairpinning_fn (vlib_main_t * vm,
1942                          vlib_node_runtime_t * node,
1943                          vlib_frame_t * frame)
1944 {
1945   return nat44_hairpinning_fn_inline (vm, node, frame, 1);
1946 }
1947
1948 VLIB_REGISTER_NODE (nat44_ed_hairpinning_node) = {
1949   .function = nat44_ed_hairpinning_fn,
1950   .name = "nat44-ed-hairpinning",
1951   .vector_size = sizeof (u32),
1952   .type = VLIB_NODE_TYPE_INTERNAL,
1953   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
1954   .error_strings = snat_in2out_error_strings,
1955   .n_next_nodes = 2,
1956   .next_nodes = {
1957     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
1958     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
1959   },
1960 };
1961
1962 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpinning_node,
1963                               nat44_ed_hairpinning_fn);
1964
1965 static inline void
1966 nat44_reass_hairpinning (snat_main_t *sm,
1967                          vlib_buffer_t * b0,
1968                          ip4_header_t * ip0,
1969                          u16 sport,
1970                          u16 dport,
1971                          u32 proto0)
1972 {
1973   snat_session_key_t key0, sm0;
1974   snat_session_t * s0;
1975   clib_bihash_kv_8_8_t kv0, value0;
1976   ip_csum_t sum0;
1977   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
1978   u16 new_dst_port0, old_dst_port0;
1979   udp_header_t * udp0;
1980   tcp_header_t * tcp0;
1981
1982   key0.addr = ip0->dst_address;
1983   key0.port = dport;
1984   key0.protocol = proto0;
1985   key0.fib_index = sm->outside_fib_index;
1986   kv0.key = key0.as_u64;
1987
1988   udp0 = ip4_next_header (ip0);
1989
1990   /* Check if destination is static mappings */
1991   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1992     {
1993       new_dst_addr0 = sm0.addr.as_u32;
1994       new_dst_port0 = sm0.port;
1995       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1996     }
1997   /* or active sessions */
1998   else
1999     {
2000       if (sm->num_workers > 1)
2001         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2002       else
2003         ti = sm->num_workers;
2004
2005       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2006         {
2007           si = value0.value;
2008           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2009           new_dst_addr0 = s0->in2out.addr.as_u32;
2010           new_dst_port0 = s0->in2out.port;
2011           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2012         }
2013     }
2014
2015   /* Destination is behind the same NAT, use internal address and port */
2016   if (new_dst_addr0)
2017     {
2018       old_dst_addr0 = ip0->dst_address.as_u32;
2019       ip0->dst_address.as_u32 = new_dst_addr0;
2020       sum0 = ip0->checksum;
2021       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2022                              ip4_header_t, dst_address);
2023       ip0->checksum = ip_csum_fold (sum0);
2024
2025       old_dst_port0 = dport;
2026       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2027                        ip4_is_first_fragment (ip0)))
2028         {
2029           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2030             {
2031               tcp0 = ip4_next_header (ip0);
2032               tcp0->dst = new_dst_port0;
2033               sum0 = tcp0->checksum;
2034               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2035                                      ip4_header_t, dst_address);
2036               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2037                                      ip4_header_t /* cheat */, length);
2038               tcp0->checksum = ip_csum_fold(sum0);
2039             }
2040           else
2041             {
2042               udp0->dst_port = new_dst_port0;
2043               udp0->checksum = 0;
2044             }
2045         }
2046       else
2047         {
2048           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2049             {
2050               tcp0 = ip4_next_header (ip0);
2051               sum0 = tcp0->checksum;
2052               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2053                                      ip4_header_t, dst_address);
2054               tcp0->checksum = ip_csum_fold(sum0);
2055             }
2056         }
2057     }
2058 }
2059
2060 static uword
2061 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2062                             vlib_node_runtime_t * node,
2063                             vlib_frame_t * frame)
2064 {
2065   u32 n_left_from, *from, *to_next;
2066   snat_in2out_next_t next_index;
2067   u32 pkts_processed = 0;
2068   snat_main_t *sm = &snat_main;
2069   f64 now = vlib_time_now (vm);
2070   u32 thread_index = vm->thread_index;
2071   snat_main_per_thread_data_t *per_thread_data =
2072     &sm->per_thread_data[thread_index];
2073   u32 *fragments_to_drop = 0;
2074   u32 *fragments_to_loopback = 0;
2075
2076   from = vlib_frame_vector_args (frame);
2077   n_left_from = frame->n_vectors;
2078   next_index = node->cached_next_index;
2079
2080   while (n_left_from > 0)
2081     {
2082       u32 n_left_to_next;
2083
2084       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2085
2086       while (n_left_from > 0 && n_left_to_next > 0)
2087        {
2088           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2089           vlib_buffer_t *b0;
2090           u32 next0;
2091           u8 cached0 = 0;
2092           ip4_header_t *ip0;
2093           nat_reass_ip4_t *reass0;
2094           udp_header_t * udp0;
2095           tcp_header_t * tcp0;
2096           snat_session_key_t key0;
2097           clib_bihash_kv_8_8_t kv0, value0;
2098           snat_session_t * s0 = 0;
2099           u16 old_port0, new_port0;
2100           ip_csum_t sum0;
2101
2102           /* speculatively enqueue b0 to the current next frame */
2103           bi0 = from[0];
2104           to_next[0] = bi0;
2105           from += 1;
2106           to_next += 1;
2107           n_left_from -= 1;
2108           n_left_to_next -= 1;
2109
2110           b0 = vlib_get_buffer (vm, bi0);
2111           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2112
2113           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2114           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2115                                                                sw_if_index0);
2116
2117           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2118             {
2119               next0 = SNAT_IN2OUT_NEXT_DROP;
2120               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2121               goto trace0;
2122             }
2123
2124           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2125           udp0 = ip4_next_header (ip0);
2126           tcp0 = (tcp_header_t *) udp0;
2127           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2128
2129           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2130                                                  ip0->dst_address,
2131                                                  ip0->fragment_id,
2132                                                  ip0->protocol,
2133                                                  1,
2134                                                  &fragments_to_drop);
2135
2136           if (PREDICT_FALSE (!reass0))
2137             {
2138               next0 = SNAT_IN2OUT_NEXT_DROP;
2139               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2140               nat_log_notice ("maximum reassemblies exceeded");
2141               goto trace0;
2142             }
2143
2144           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2145             {
2146               key0.addr = ip0->src_address;
2147               key0.port = udp0->src_port;
2148               key0.protocol = proto0;
2149               key0.fib_index = rx_fib_index0;
2150               kv0.key = key0.as_u64;
2151
2152               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2153                 {
2154                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2155                       ip0, proto0, rx_fib_index0, thread_index)))
2156                     goto trace0;
2157
2158                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2159                                      &s0, node, next0, thread_index);
2160
2161                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2162                     goto trace0;
2163
2164                   reass0->sess_index = s0 - per_thread_data->sessions;
2165                 }
2166               else
2167                 {
2168                   s0 = pool_elt_at_index (per_thread_data->sessions,
2169                                           value0.value);
2170                   reass0->sess_index = value0.value;
2171                 }
2172               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2173             }
2174           else
2175             {
2176               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2177                 {
2178                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2179                     {
2180                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2181                       nat_log_notice ("maximum fragments per reassembly exceeded");
2182                       next0 = SNAT_IN2OUT_NEXT_DROP;
2183                       goto trace0;
2184                     }
2185                   cached0 = 1;
2186                   goto trace0;
2187                 }
2188               s0 = pool_elt_at_index (per_thread_data->sessions,
2189                                       reass0->sess_index);
2190             }
2191
2192           old_addr0 = ip0->src_address.as_u32;
2193           ip0->src_address = s0->out2in.addr;
2194           new_addr0 = ip0->src_address.as_u32;
2195           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2196
2197           sum0 = ip0->checksum;
2198           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2199                                  ip4_header_t,
2200                                  src_address /* changed member */);
2201           ip0->checksum = ip_csum_fold (sum0);
2202
2203           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2204             {
2205               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2206                 {
2207                   old_port0 = tcp0->src_port;
2208                   tcp0->src_port = s0->out2in.port;
2209                   new_port0 = tcp0->src_port;
2210
2211                   sum0 = tcp0->checksum;
2212                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2213                                          ip4_header_t,
2214                                          dst_address /* changed member */);
2215                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2216                                          ip4_header_t /* cheat */,
2217                                          length /* changed member */);
2218                   tcp0->checksum = ip_csum_fold(sum0);
2219                 }
2220               else
2221                 {
2222                   old_port0 = udp0->src_port;
2223                   udp0->src_port = s0->out2in.port;
2224                   udp0->checksum = 0;
2225                 }
2226             }
2227
2228           /* Hairpinning */
2229           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2230                                    s0->ext_host_port, proto0);
2231
2232           /* Accounting */
2233           nat44_session_update_counters (s0, now,
2234                                          vlib_buffer_length_in_chain (vm, b0));
2235           /* Per-user LRU list maintenance */
2236           nat44_session_update_lru (sm, s0, thread_index);
2237
2238         trace0:
2239           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2240                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2241             {
2242               nat44_in2out_reass_trace_t *t =
2243                  vlib_add_trace (vm, node, b0, sizeof (*t));
2244               t->cached = cached0;
2245               t->sw_if_index = sw_if_index0;
2246               t->next_index = next0;
2247             }
2248
2249           if (cached0)
2250             {
2251               n_left_to_next++;
2252               to_next--;
2253             }
2254           else
2255             {
2256               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2257
2258               /* verify speculative enqueue, maybe switch current next frame */
2259               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2260                                                to_next, n_left_to_next,
2261                                                bi0, next0);
2262             }
2263
2264           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2265             {
2266               from = vlib_frame_vector_args (frame);
2267               u32 len = vec_len (fragments_to_loopback);
2268               if (len <= VLIB_FRAME_SIZE)
2269                 {
2270                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2271                   n_left_from = len;
2272                   vec_reset_length (fragments_to_loopback);
2273                 }
2274               else
2275                 {
2276                   clib_memcpy (from,
2277                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2278                                sizeof (u32) * VLIB_FRAME_SIZE);
2279                   n_left_from = VLIB_FRAME_SIZE;
2280                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2281                 }
2282             }
2283        }
2284
2285       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2286     }
2287
2288   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2289                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2290                                pkts_processed);
2291
2292   nat_send_all_to_node (vm, fragments_to_drop, node,
2293                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2294                         SNAT_IN2OUT_NEXT_DROP);
2295
2296   vec_free (fragments_to_drop);
2297   vec_free (fragments_to_loopback);
2298   return frame->n_vectors;
2299 }
2300
2301 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2302   .function = nat44_in2out_reass_node_fn,
2303   .name = "nat44-in2out-reass",
2304   .vector_size = sizeof (u32),
2305   .format_trace = format_nat44_in2out_reass_trace,
2306   .type = VLIB_NODE_TYPE_INTERNAL,
2307
2308   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2309   .error_strings = snat_in2out_error_strings,
2310
2311   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2312   .next_nodes = {
2313     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2314     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2315     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2316     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2317     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2318   },
2319 };
2320
2321 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2322                               nat44_in2out_reass_node_fn);
2323
2324 /*******************************/
2325 /*** endpoint-dependent mode ***/
2326 /*******************************/
2327
2328 static_always_inline int
2329 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
2330 {
2331   icmp46_header_t *icmp0;
2332   nat_ed_ses_key_t key0;
2333   icmp_echo_header_t *echo0, *inner_echo0 = 0;
2334   ip4_header_t *inner_ip0 = 0;
2335   void *l4_header = 0;
2336   icmp46_header_t *inner_icmp0;
2337
2338   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
2339   echo0 = (icmp_echo_header_t *)(icmp0+1);
2340
2341   if (!icmp_is_error_message (icmp0))
2342     {
2343       key0.proto = IP_PROTOCOL_ICMP;
2344       key0.l_addr = ip0->src_address;
2345       key0.r_addr = ip0->dst_address;
2346       key0.l_port = echo0->identifier;
2347       key0.r_port = 0;
2348     }
2349   else
2350     {
2351       inner_ip0 = (ip4_header_t *)(echo0+1);
2352       l4_header = ip4_next_header (inner_ip0);
2353       key0.proto = inner_ip0->protocol;
2354       key0.r_addr = inner_ip0->src_address;
2355       key0.l_addr = inner_ip0->dst_address;
2356       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
2357         {
2358         case SNAT_PROTOCOL_ICMP:
2359           inner_icmp0 = (icmp46_header_t*)l4_header;
2360           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
2361           key0.r_port = 0;
2362           key0.l_port = inner_echo0->identifier;
2363           break;
2364         case SNAT_PROTOCOL_UDP:
2365         case SNAT_PROTOCOL_TCP:
2366           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
2367           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
2368           break;
2369         default:
2370           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
2371         }
2372     }
2373   *p_key0 = key0;
2374   return 0;
2375 }
2376
2377 static u32
2378 slow_path_ed (snat_main_t *sm,
2379               vlib_buffer_t *b,
2380               u32 rx_fib_index,
2381               clib_bihash_kv_16_8_t *kv,
2382               snat_session_t ** sessionp,
2383               vlib_node_runtime_t * node,
2384               u32 next,
2385               u32 thread_index)
2386 {
2387   snat_session_t *s;
2388   snat_user_t *u;
2389   snat_session_key_t key0, key1;
2390   u8 lb = 0, is_sm = 0;
2391   u32 address_index = ~0;
2392   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2393   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
2394   u32 proto = ip_proto_to_snat_proto (key->proto);
2395   nat_outside_fib_t *outside_fib;
2396   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2397   fib_prefix_t pfx = {
2398     .fp_proto = FIB_PROTOCOL_IP4,
2399     .fp_len = 32,
2400     .fp_addr = {
2401         .ip4.as_u32 = key->r_addr.as_u32,
2402     },
2403   };
2404
2405   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
2406     {
2407       b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2408       nat_ipfix_logging_max_sessions(sm->max_translations);
2409       nat_log_notice ("maximum sessions exceeded");
2410       return SNAT_IN2OUT_NEXT_DROP;
2411     }
2412
2413   key0.addr = key->l_addr;
2414   key0.port = key->l_port;
2415   key1.protocol = key0.protocol = proto;
2416   key0.fib_index = rx_fib_index;
2417   key1.fib_index = sm->outside_fib_index;
2418   /* First try to match static mapping by local address and port */
2419   if (snat_static_mapping_match (sm, key0, &key1, 0, 0, 0, &lb))
2420     {
2421       /* Try to create dynamic translation */
2422       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index,
2423                                                thread_index, &key1,
2424                                                &address_index,
2425                                                sm->port_per_thread,
2426                                                tsm->snat_thread_index))
2427         {
2428           nat_log_notice ("addresses exhausted");
2429           b->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
2430           return SNAT_IN2OUT_NEXT_DROP;
2431         }
2432     }
2433   else
2434     is_sm = 1;
2435
2436   u = nat_user_get_or_create (sm, &key->l_addr, rx_fib_index, thread_index);
2437   if (!u)
2438     {
2439       nat_log_warn ("create NAT user failed");
2440       return SNAT_IN2OUT_NEXT_DROP;
2441     }
2442
2443   s = nat_session_alloc_or_recycle (sm, u, thread_index);
2444   if (!s)
2445     {
2446       nat44_delete_user_with_no_session (sm, u, thread_index);
2447       nat_log_warn ("create NAT session failed");
2448       return SNAT_IN2OUT_NEXT_DROP;
2449     }
2450
2451   user_session_increment (sm, u, is_sm);
2452   if (is_sm)
2453     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2454   if (lb)
2455     s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
2456   s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2457   s->outside_address_index = address_index;
2458   s->ext_host_addr = key->r_addr;
2459   s->ext_host_port = key->r_port;
2460   s->in2out = key0;
2461   s->out2in = key1;
2462   s->out2in.protocol = key0.protocol;
2463
2464   switch (vec_len (sm->outside_fibs))
2465     {
2466     case 0:
2467       s->out2in.fib_index = sm->outside_fib_index;
2468       break;
2469     case 1:
2470       s->out2in.fib_index = sm->outside_fibs[0].fib_index;
2471       break;
2472     default:
2473       vec_foreach (outside_fib, sm->outside_fibs)
2474         {
2475           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2476           if (FIB_NODE_INDEX_INVALID != fei)
2477             {
2478               if (fib_entry_get_resolving_interface (fei) != ~0)
2479                 {
2480                   s->out2in.fib_index = outside_fib->fib_index;
2481                   break;
2482                 }
2483             }
2484         }
2485       break;
2486     }
2487
2488   /* Add to lookup tables */
2489   kv->value = s - tsm->sessions;
2490   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, kv, 1))
2491     nat_log_notice ("in2out-ed key add failed");
2492
2493   make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
2494               key1.port, key->r_port);
2495   kv->value = s - tsm->sessions;
2496   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, kv, 1))
2497     nat_log_notice ("out2in-ed key add failed");
2498
2499   *sessionp = s;
2500
2501   /* log NAT event */
2502   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
2503                                       s->out2in.addr.as_u32,
2504                                       s->in2out.protocol,
2505                                       s->in2out.port,
2506                                       s->out2in.port,
2507                                       s->in2out.fib_index);
2508   return next;
2509 }
2510
2511 static_always_inline int
2512 nat44_ed_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
2513                         u32 sw_if_index, ip4_header_t * ip, u32 proto,
2514                         u32 rx_fib_index, u32 thread_index)
2515 {
2516   udp_header_t *udp = ip4_next_header (ip);
2517   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2518   clib_bihash_kv_16_8_t kv, value;
2519   snat_session_key_t key0, key1;
2520
2521   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, ip->protocol,
2522               sm->outside_fib_index, udp->dst_port, udp->src_port);
2523
2524   /* NAT packet aimed at external address if */
2525   /* has active sessions */
2526   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2527     {
2528       key0.addr = ip->dst_address;
2529       key0.port = udp->dst_port;
2530       key0.protocol = proto;
2531       key0.fib_index = sm->outside_fib_index;
2532       /* or is static mappings */
2533       if (!snat_static_mapping_match(sm, key0, &key1, 1, 0, 0, 0))
2534         return 0;
2535     }
2536   else
2537     return 0;
2538
2539   if (sm->forwarding_enabled)
2540     return 1;
2541
2542   return snat_not_translate_fast(sm, node, sw_if_index, ip, proto, rx_fib_index);
2543 }
2544
2545 static_always_inline int
2546 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
2547                                       u32 thread_index, f64 now,
2548                                       vlib_main_t * vm, vlib_buffer_t * b)
2549 {
2550   nat_ed_ses_key_t key;
2551   clib_bihash_kv_16_8_t kv, value;
2552   udp_header_t *udp;
2553   snat_session_t *s = 0;
2554   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2555
2556   if (!sm->forwarding_enabled)
2557     return 0;
2558
2559   if (ip->protocol == IP_PROTOCOL_ICMP)
2560     {
2561       key.as_u64[0] = key.as_u64[1] = 0;
2562       if (icmp_get_ed_key (ip, &key))
2563         return 0;
2564       key.fib_index = 0;
2565       kv.key[0] = key.as_u64[0];
2566       kv.key[1] = key.as_u64[1];
2567     }
2568   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
2569     {
2570       udp = ip4_next_header(ip);
2571       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0,
2572                   udp->src_port, udp->dst_port);
2573     }
2574   else
2575     {
2576       make_ed_kv (&kv, &ip->src_address, &ip->dst_address, ip->protocol, 0, 0,
2577                   0);
2578     }
2579
2580   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2581     {
2582       s = pool_elt_at_index (tsm->sessions, value.value);
2583       if (is_fwd_bypass_session (s))
2584         {
2585           if (ip->protocol == IP_PROTOCOL_TCP)
2586             {
2587               tcp_header_t *tcp = ip4_next_header(ip);
2588               if (nat44_set_tcp_session_state_i2o (sm, s, tcp, thread_index))
2589                 return 1;
2590             }
2591           /* Per-user LRU list maintenance */
2592           nat44_session_update_lru (sm, s, thread_index);
2593           /* Accounting */
2594           nat44_session_update_counters (s, now,
2595                                          vlib_buffer_length_in_chain (vm, b));
2596           return 1;
2597         }
2598       else
2599         return 0;
2600     }
2601
2602   return 0;
2603 }
2604
2605 static_always_inline int
2606 nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
2607                                        u8 proto, u16 src_port, u16 dst_port,
2608                                        u32 thread_index, u32 sw_if_index)
2609 {
2610   clib_bihash_kv_16_8_t kv, value;
2611   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2612   snat_interface_t *i;
2613   snat_session_t *s;
2614   u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2615
2616   /* src NAT check */
2617   make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, fib_index,
2618               src_port, dst_port);
2619   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
2620     return 1;
2621
2622   /* dst NAT check */
2623   make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, fib_index,
2624               dst_port, src_port);
2625   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2626   {
2627     s = pool_elt_at_index (tsm->sessions, value.value);
2628     if (is_fwd_bypass_session (s))
2629       return 0;
2630
2631     /* hairpinning */
2632     pool_foreach (i, sm->output_feature_interfaces,
2633     ({
2634       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
2635         return 0;
2636     }));
2637     return 1;
2638   }
2639
2640   return 0;
2641 }
2642
2643 u32
2644 icmp_match_in2out_ed(snat_main_t *sm, vlib_node_runtime_t *node,
2645                      u32 thread_index, vlib_buffer_t *b, ip4_header_t *ip,
2646                      u8 *p_proto, snat_session_key_t *p_value,
2647                      u8 *p_dont_translate, void *d, void *e)
2648 {
2649   icmp46_header_t *icmp;
2650   u32 sw_if_index;
2651   u32 rx_fib_index;
2652   nat_ed_ses_key_t key;
2653   snat_session_t *s = 0;
2654   u8 dont_translate = 0;
2655   clib_bihash_kv_16_8_t kv, value;
2656   u32 next = ~0;
2657   int err;
2658   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2659
2660   icmp = (icmp46_header_t *) ip4_next_header (ip);
2661   sw_if_index = vnet_buffer(b)->sw_if_index[VLIB_RX];
2662   rx_fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
2663
2664   key.as_u64[0] = key.as_u64[1] = 0;
2665   err = icmp_get_ed_key (ip, &key);
2666   if (err != 0)
2667     {
2668       b->error = node->errors[err];
2669       next = SNAT_IN2OUT_NEXT_DROP;
2670       goto out;
2671     }
2672   key.fib_index = rx_fib_index;
2673
2674   kv.key[0] = key.as_u64[0];
2675   kv.key[1] = key.as_u64[1];
2676
2677   if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
2678     {
2679       if (vnet_buffer(b)->sw_if_index[VLIB_TX] != ~0)
2680         {
2681           if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(sm, ip,
2682               key.proto, key.l_port, key.r_port, thread_index, sw_if_index)))
2683             {
2684               dont_translate = 1;
2685               goto out;
2686             }
2687         }
2688       else
2689         {
2690           if (PREDICT_FALSE(nat44_ed_not_translate(sm, node, sw_if_index,
2691               ip, SNAT_PROTOCOL_ICMP, rx_fib_index, thread_index)))
2692             {
2693               dont_translate = 1;
2694               goto out;
2695             }
2696         }
2697
2698       if (PREDICT_FALSE(icmp_is_error_message (icmp)))
2699         {
2700           b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2701           next = SNAT_IN2OUT_NEXT_DROP;
2702           goto out;
2703         }
2704
2705       next = slow_path_ed (sm, b, rx_fib_index, &kv, &s, node, next,
2706                            thread_index);
2707
2708       if (PREDICT_FALSE (next == SNAT_IN2OUT_NEXT_DROP))
2709         goto out;
2710     }
2711   else
2712     {
2713       if (PREDICT_FALSE(icmp->type != ICMP4_echo_request &&
2714                         icmp->type != ICMP4_echo_reply &&
2715                         !icmp_is_error_message (icmp)))
2716         {
2717           b->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
2718           next = SNAT_IN2OUT_NEXT_DROP;
2719           goto out;
2720         }
2721
2722       s = pool_elt_at_index (tsm->sessions, value.value);
2723     }
2724
2725   *p_proto = ip_proto_to_snat_proto (key.proto);
2726 out:
2727   if (s)
2728     *p_value = s->out2in;
2729   *p_dont_translate = dont_translate;
2730   if (d)
2731     *(snat_session_t**)d = s;
2732   return next;
2733 }
2734
2735 static inline void
2736 nat44_ed_hairpinning_unknown_proto (snat_main_t *sm,
2737                                     vlib_buffer_t * b,
2738                                     ip4_header_t * ip)
2739 {
2740   u32 old_addr, new_addr = 0, ti = 0;
2741   clib_bihash_kv_8_8_t kv, value;
2742   clib_bihash_kv_16_8_t s_kv, s_value;
2743   snat_static_mapping_t *m;
2744   ip_csum_t sum;
2745   snat_session_t *s;
2746   snat_main_per_thread_data_t *tsm;
2747
2748   if (sm->num_workers > 1)
2749     ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
2750   else
2751     ti = sm->num_workers;
2752   tsm = &sm->per_thread_data[ti];
2753
2754   old_addr = ip->dst_address.as_u32;
2755   make_ed_kv (&s_kv, &ip->dst_address, &ip->src_address, ip->protocol,
2756               sm->outside_fib_index, 0, 0);
2757   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2758     {
2759       make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
2760       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
2761         return;
2762
2763       m = pool_elt_at_index (sm->static_mappings, value.value);
2764       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2765         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
2766       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
2767     }
2768   else
2769     {
2770       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
2771       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2772         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
2773       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
2774     }
2775   sum = ip->checksum;
2776   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
2777   ip->checksum = ip_csum_fold (sum);
2778 }
2779
2780 static snat_session_t *
2781 nat44_ed_in2out_unknown_proto (snat_main_t *sm,
2782                                vlib_buffer_t * b,
2783                                ip4_header_t * ip,
2784                                u32 rx_fib_index,
2785                                u32 thread_index,
2786                                f64 now,
2787                                vlib_main_t * vm,
2788                                vlib_node_runtime_t * node)
2789 {
2790   clib_bihash_kv_8_8_t kv, value;
2791   clib_bihash_kv_16_8_t s_kv, s_value;
2792   snat_static_mapping_t *m;
2793   u32 old_addr, new_addr = 0;
2794   ip_csum_t sum;
2795   snat_user_t *u;
2796   dlist_elt_t *head, *elt;
2797   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2798   u32 elt_index, head_index, ses_index;
2799   snat_session_t * s;
2800   u32 address_index = ~0, outside_fib_index = sm->outside_fib_index;
2801   int i;
2802   u8 is_sm = 0;
2803   nat_outside_fib_t *outside_fib;
2804   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
2805   fib_prefix_t pfx = {
2806     .fp_proto = FIB_PROTOCOL_IP4,
2807     .fp_len = 32,
2808     .fp_addr = {
2809         .ip4.as_u32 = ip->dst_address.as_u32,
2810     },
2811   };
2812
2813   switch (vec_len (sm->outside_fibs))
2814     {
2815     case 0:
2816       outside_fib_index = sm->outside_fib_index;
2817       break;
2818     case 1:
2819       outside_fib_index = sm->outside_fibs[0].fib_index;
2820       break;
2821     default:
2822       vec_foreach (outside_fib, sm->outside_fibs)
2823         {
2824           fei = fib_table_lookup (outside_fib->fib_index, &pfx);
2825           if (FIB_NODE_INDEX_INVALID != fei)
2826             {
2827               if (fib_entry_get_resolving_interface (fei) != ~0)
2828                 {
2829                   outside_fib_index = outside_fib->fib_index;
2830                   break;
2831                 }
2832             }
2833         }
2834       break;
2835     }
2836   old_addr = ip->src_address.as_u32;
2837
2838   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
2839               rx_fib_index, 0, 0);
2840
2841   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &s_kv, &s_value))
2842     {
2843       s = pool_elt_at_index (tsm->sessions, s_value.value);
2844       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
2845     }
2846   else
2847     {
2848       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
2849         {
2850           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
2851           nat_ipfix_logging_max_sessions(sm->max_translations);
2852           nat_log_notice ("maximum sessions exceeded");
2853           return 0;
2854         }
2855
2856       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
2857                                   thread_index);
2858       if (!u)
2859         {
2860           nat_log_warn ("create NAT user failed");
2861           return 0;
2862         }
2863
2864       make_sm_kv (&kv, &ip->src_address, 0, rx_fib_index, 0);
2865
2866       /* Try to find static mapping first */
2867       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
2868         {
2869           m = pool_elt_at_index (sm->static_mappings, value.value);
2870           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
2871           is_sm = 1;
2872           goto create_ses;
2873         }
2874       /* Fallback to 3-tuple key */
2875       else
2876         {
2877           /* Choose same out address as for TCP/UDP session to same destination */
2878           head_index = u->sessions_per_user_list_head_index;
2879           head = pool_elt_at_index (tsm->list_pool, head_index);
2880           elt_index = head->next;
2881           if (PREDICT_FALSE (elt_index == ~0))
2882             ses_index = ~0;
2883           else
2884             {
2885               elt = pool_elt_at_index (tsm->list_pool, elt_index);
2886               ses_index = elt->value;
2887             }
2888
2889           while (ses_index != ~0)
2890             {
2891               s =  pool_elt_at_index (tsm->sessions, ses_index);
2892               elt_index = elt->next;
2893               elt = pool_elt_at_index (tsm->list_pool, elt_index);
2894               ses_index = elt->value;
2895
2896               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
2897                 {
2898                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
2899                   address_index = s->outside_address_index;
2900
2901                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
2902                               ip->protocol, outside_fib_index, 0, 0);
2903                   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2904                     goto create_ses;
2905
2906                   break;
2907                 }
2908             }
2909
2910           for (i = 0; i < vec_len (sm->addresses); i++)
2911             {
2912               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
2913                           ip->protocol, outside_fib_index, 0, 0);
2914               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
2915                 {
2916                   new_addr = ip->src_address.as_u32 =
2917                     sm->addresses[i].addr.as_u32;
2918                   address_index = i;
2919                   goto create_ses;
2920                 }
2921             }
2922           return 0;
2923         }
2924
2925 create_ses:
2926       s = nat_session_alloc_or_recycle (sm, u, thread_index);
2927       if (!s)
2928         {
2929           nat44_delete_user_with_no_session (sm, u, thread_index);
2930           nat_log_warn ("create NAT session failed");
2931           return 0;
2932         }
2933
2934       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
2935       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
2936       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
2937       s->outside_address_index = address_index;
2938       s->out2in.addr.as_u32 = new_addr;
2939       s->out2in.fib_index = outside_fib_index;
2940       s->in2out.addr.as_u32 = old_addr;
2941       s->in2out.fib_index = rx_fib_index;
2942       s->in2out.port = s->out2in.port = ip->protocol;
2943       if (is_sm)
2944         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
2945       user_session_increment (sm, u, is_sm);
2946
2947       /* Add to lookup tables */
2948       make_ed_kv (&s_kv, &s->in2out.addr, &ip->dst_address, ip->protocol,
2949                   rx_fib_index, 0, 0);
2950       s_kv.value = s - tsm->sessions;
2951       if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &s_kv, 1))
2952         nat_log_notice ("in2out key add failed");
2953
2954       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
2955                   outside_fib_index, 0, 0);
2956       s_kv.value = s - tsm->sessions;
2957       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
2958         nat_log_notice ("out2in key add failed");
2959   }
2960
2961   /* Update IP checksum */
2962   sum = ip->checksum;
2963   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
2964   ip->checksum = ip_csum_fold (sum);
2965
2966   /* Accounting */
2967   nat44_session_update_counters (s, now, vlib_buffer_length_in_chain (vm, b));
2968   /* Per-user LRU list maintenance */
2969   nat44_session_update_lru (sm, s, thread_index);
2970
2971   /* Hairpinning */
2972   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2973     nat44_ed_hairpinning_unknown_proto(sm, b, ip);
2974
2975   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
2976     vnet_buffer(b)->sw_if_index[VLIB_TX] = outside_fib_index;
2977
2978   return s;
2979 }
2980
2981 static inline uword
2982 nat44_ed_in2out_node_fn_inline (vlib_main_t * vm,
2983                                 vlib_node_runtime_t * node,
2984                                 vlib_frame_t * frame, int is_slow_path,
2985                                 int is_output_feature)
2986 {
2987   u32 n_left_from, *from, *to_next, pkts_processed = 0, stats_node_index;
2988   snat_in2out_next_t next_index;
2989   snat_main_t *sm = &snat_main;
2990   f64 now = vlib_time_now (vm);
2991   u32 thread_index = vm->thread_index;
2992   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
2993
2994   stats_node_index = is_slow_path ? nat44_ed_in2out_slowpath_node.index :
2995     nat44_ed_in2out_node.index;
2996
2997   from = vlib_frame_vector_args (frame);
2998   n_left_from = frame->n_vectors;
2999   next_index = node->cached_next_index;
3000
3001   while (n_left_from > 0)
3002     {
3003       u32 n_left_to_next;
3004
3005       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
3006
3007       while (n_left_from >= 4 && n_left_to_next >= 2)
3008         {
3009           u32 bi0, bi1;
3010           vlib_buffer_t *b0, *b1;
3011           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3012               new_addr0, old_addr0;
3013           u32 next1, sw_if_index1, rx_fib_index1, iph_offset1 = 0, proto1,
3014               new_addr1, old_addr1;
3015           u16 old_port0, new_port0, old_port1, new_port1;
3016           ip4_header_t *ip0, *ip1;
3017           udp_header_t *udp0, *udp1;
3018           tcp_header_t *tcp0, *tcp1;
3019           icmp46_header_t *icmp0, *icmp1;
3020           snat_session_t *s0 = 0, *s1 = 0;
3021           clib_bihash_kv_16_8_t kv0, value0, kv1, value1;
3022           ip_csum_t sum0, sum1;
3023
3024           /* Prefetch next iteration. */
3025           {
3026             vlib_buffer_t * p2, * p3;
3027
3028             p2 = vlib_get_buffer (vm, from[2]);
3029             p3 = vlib_get_buffer (vm, from[3]);
3030
3031             vlib_prefetch_buffer_header (p2, LOAD);
3032             vlib_prefetch_buffer_header (p3, LOAD);
3033
3034             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3035             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3036           }
3037
3038           /* speculatively enqueue b0 and b1 to the current next frame */
3039           to_next[0] = bi0 = from[0];
3040           to_next[1] = bi1 = from[1];
3041           from += 2;
3042           to_next += 2;
3043           n_left_from -= 2;
3044           n_left_to_next -= 2;
3045
3046           b0 = vlib_get_buffer (vm, bi0);
3047           b1 = vlib_get_buffer (vm, bi1);
3048
3049           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3050
3051           if (is_output_feature)
3052             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3053
3054           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3055                  iph_offset0);
3056
3057           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3058           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3059                                                                sw_if_index0);
3060
3061           if (PREDICT_FALSE(ip0->ttl == 1))
3062             {
3063               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3064               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3065                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3066                                            0);
3067               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3068               goto trace00;
3069             }
3070
3071           udp0 = ip4_next_header (ip0);
3072           tcp0 = (tcp_header_t *) udp0;
3073           icmp0 = (icmp46_header_t *) udp0;
3074           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3075
3076           if (is_slow_path)
3077             {
3078               if (PREDICT_FALSE (proto0 == ~0))
3079                 {
3080                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3081                                                       rx_fib_index0,
3082                                                       thread_index, now, vm,
3083                                                       node);
3084                   if (!s0)
3085                     next0 = SNAT_IN2OUT_NEXT_DROP;
3086                   goto trace00;
3087                 }
3088
3089               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3090                 {
3091                   next0 = icmp_in2out_slow_path
3092                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3093                      next0, now, thread_index, &s0);
3094                   goto trace00;
3095                 }
3096             }
3097           else
3098             {
3099                if (is_output_feature)
3100                 {
3101                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3102                       sm, ip0, thread_index, now, vm, b0)))
3103                     goto trace00;
3104                 }
3105
3106               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3107                 {
3108                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3109                   goto trace00;
3110                 }
3111
3112               if (ip4_is_fragment (ip0))
3113                 {
3114                   b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3115                   next0 = SNAT_IN2OUT_NEXT_DROP;
3116                   goto trace00;
3117                 }
3118             }
3119
3120           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3121                       rx_fib_index0, udp0->src_port, udp0->dst_port);
3122
3123           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3124             {
3125               if (is_slow_path)
3126                 {
3127                   if (is_output_feature)
3128                     {
3129                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3130                           sm, ip0, ip0->protocol, udp0->src_port,
3131                           udp0->dst_port, thread_index, sw_if_index0)))
3132                         goto trace00;
3133                     }
3134                   else
3135                     {
3136                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3137                           sw_if_index0, ip0, proto0, rx_fib_index0,
3138                           thread_index)))
3139                         goto trace00;
3140                     }
3141
3142                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3143                                         next0, thread_index);
3144
3145                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3146                     goto trace00;
3147                 }
3148               else
3149                 {
3150                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3151                   goto trace00;
3152                 }
3153             }
3154           else
3155             {
3156               s0 = pool_elt_at_index (tsm->sessions, value0.value);
3157             }
3158
3159           b0->flags |= VNET_BUFFER_F_IS_NATED;
3160
3161           if (!is_output_feature)
3162             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3163
3164           old_addr0 = ip0->src_address.as_u32;
3165           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3166           sum0 = ip0->checksum;
3167           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3168                                  src_address);
3169           if (PREDICT_FALSE (is_twice_nat_session (s0)))
3170             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3171                                    s0->ext_host_addr.as_u32, ip4_header_t,
3172                                    dst_address);
3173           ip0->checksum = ip_csum_fold (sum0);
3174
3175           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3176             {
3177               old_port0 = tcp0->src_port;
3178               new_port0 = tcp0->src_port = s0->out2in.port;
3179
3180               sum0 = tcp0->checksum;
3181               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3182                                      dst_address);
3183               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3184                                      length);
3185               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3186                 {
3187                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3188                                          s0->ext_host_addr.as_u32,
3189                                          ip4_header_t, dst_address);
3190                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
3191                                          s0->ext_host_port, ip4_header_t,
3192                                          length);
3193                   tcp0->dst_port = s0->ext_host_port;
3194                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3195                 }
3196               tcp0->checksum = ip_csum_fold(sum0);
3197               if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3198                 goto trace00;
3199             }
3200           else
3201             {
3202               udp0->src_port = s0->out2in.port;
3203               udp0->checksum = 0;
3204               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3205                 {
3206                   udp0->dst_port = s0->ext_host_port;
3207                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3208                 }
3209             }
3210
3211           /* Accounting */
3212           nat44_session_update_counters (s0, now,
3213                                          vlib_buffer_length_in_chain (vm, b0));
3214           /* Per-user LRU list maintenance */
3215           nat44_session_update_lru (sm, s0, thread_index);
3216
3217         trace00:
3218           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3219                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3220             {
3221               snat_in2out_trace_t *t =
3222                 vlib_add_trace (vm, node, b0, sizeof (*t));
3223               t->is_slow_path = is_slow_path;
3224               t->sw_if_index = sw_if_index0;
3225               t->next_index = next0;
3226               t->session_index = ~0;
3227               if (s0)
3228                 t->session_index = s0 - tsm->sessions;
3229             }
3230
3231           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3232
3233
3234           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3235
3236           if (is_output_feature)
3237             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
3238
3239           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
3240                  iph_offset1);
3241
3242           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3243           rx_fib_index1 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3244                                                                sw_if_index1);
3245
3246           if (PREDICT_FALSE(ip1->ttl == 1))
3247             {
3248               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3249               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3250                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3251                                            0);
3252               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3253               goto trace01;
3254             }
3255
3256           udp1 = ip4_next_header (ip1);
3257           tcp1 = (tcp_header_t *) udp1;
3258           icmp1 = (icmp46_header_t *) udp1;
3259           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3260
3261           if (is_slow_path)
3262             {
3263               if (PREDICT_FALSE (proto1 == ~0))
3264                 {
3265                   s1 = nat44_ed_in2out_unknown_proto (sm, b1, ip1,
3266                                                       rx_fib_index1,
3267                                                       thread_index, now, vm,
3268                                                       node);
3269                   if (!s1)
3270                     next1 = SNAT_IN2OUT_NEXT_DROP;
3271                   goto trace01;
3272                 }
3273
3274               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
3275                 {
3276                   next1 = icmp_in2out_slow_path
3277                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
3278                      next1, now, thread_index, &s1);
3279                   goto trace01;
3280                 }
3281             }
3282           else
3283             {
3284                if (is_output_feature)
3285                 {
3286                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3287                       sm, ip1, thread_index, now, vm, b1)))
3288                     goto trace01;
3289                 }
3290
3291               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
3292                 {
3293                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3294                   goto trace01;
3295                 }
3296
3297               if (ip4_is_fragment (ip1))
3298                 {
3299                   b1->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3300                   next1 = SNAT_IN2OUT_NEXT_DROP;
3301                   goto trace01;
3302                 }
3303             }
3304
3305           make_ed_kv (&kv1, &ip1->src_address, &ip1->dst_address, ip1->protocol,
3306                       rx_fib_index1, udp1->src_port, udp1->dst_port);
3307
3308           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv1, &value1))
3309             {
3310               if (is_slow_path)
3311                 {
3312                   if (is_output_feature)
3313                     {
3314                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3315                           sm, ip1, ip1->protocol, udp1->src_port,
3316                           udp1->dst_port, thread_index, sw_if_index1)))
3317                         goto trace01;
3318                     }
3319                   else
3320                     {
3321                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3322                           sw_if_index1, ip1, proto1, rx_fib_index1,
3323                           thread_index)))
3324                         goto trace01;
3325                     }
3326
3327                   next1 = slow_path_ed (sm, b1, rx_fib_index1, &kv1, &s1, node,
3328                                         next1, thread_index);
3329
3330                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
3331                     goto trace01;
3332                 }
3333               else
3334                 {
3335                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3336                   goto trace01;
3337                 }
3338             }
3339           else
3340             {
3341               s1 = pool_elt_at_index (tsm->sessions, value1.value);
3342             }
3343
3344           b1->flags |= VNET_BUFFER_F_IS_NATED;
3345
3346           if (!is_output_feature)
3347             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
3348
3349           old_addr1 = ip1->src_address.as_u32;
3350           new_addr1 = ip1->src_address.as_u32 = s1->out2in.addr.as_u32;
3351           sum1 = ip1->checksum;
3352           sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3353                                  src_address);
3354           if (PREDICT_FALSE (is_twice_nat_session (s1)))
3355             sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3356                                    s1->ext_host_addr.as_u32, ip4_header_t,
3357                                    dst_address);
3358           ip1->checksum = ip_csum_fold (sum1);
3359
3360           if (PREDICT_TRUE (proto1 == SNAT_PROTOCOL_TCP))
3361             {
3362               old_port1 = tcp1->src_port;
3363               new_port1 = tcp1->src_port = s1->out2in.port;
3364
3365               sum1 = tcp1->checksum;
3366               sum1 = ip_csum_update (sum1, old_addr1, new_addr1, ip4_header_t,
3367                                      dst_address);
3368               sum1 = ip_csum_update (sum1, old_port1, new_port1, ip4_header_t,
3369                                      length);
3370               if (PREDICT_FALSE (is_twice_nat_session (s1)))
3371                 {
3372                   sum1 = ip_csum_update (sum1, ip1->dst_address.as_u32,
3373                                          s1->ext_host_addr.as_u32,
3374                                          ip4_header_t, dst_address);
3375                   sum1 = ip_csum_update (sum1, tcp1->dst_port,
3376                                          s1->ext_host_port, ip4_header_t,
3377                                          length);
3378                   tcp1->dst_port = s1->ext_host_port;
3379                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3380                 }
3381               tcp1->checksum = ip_csum_fold(sum1);
3382               if (nat44_set_tcp_session_state_i2o (sm, s1, tcp1, thread_index))
3383                 goto trace01;
3384             }
3385           else
3386             {
3387               udp1->src_port = s1->out2in.port;
3388               udp1->checksum = 0;
3389               if (PREDICT_FALSE (is_twice_nat_session (s1)))
3390                 {
3391                   udp1->dst_port = s1->ext_host_port;
3392                   ip1->dst_address.as_u32 = s1->ext_host_addr.as_u32;
3393                 }
3394             }
3395
3396           /* Accounting */
3397           nat44_session_update_counters (s1, now,
3398                                          vlib_buffer_length_in_chain (vm, b1));
3399           /* Per-user LRU list maintenance */
3400           nat44_session_update_lru (sm, s1, thread_index);
3401
3402         trace01:
3403           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3404                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3405             {
3406               snat_in2out_trace_t *t =
3407                 vlib_add_trace (vm, node, b1, sizeof (*t));
3408               t->is_slow_path = is_slow_path;
3409               t->sw_if_index = sw_if_index1;
3410               t->next_index = next1;
3411               t->session_index = ~0;
3412               if (s1)
3413                 t->session_index = s1 - tsm->sessions;
3414             }
3415
3416           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3417
3418           /* verify speculative enqueues, maybe switch current next frame */
3419           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3420                                            to_next, n_left_to_next,
3421                                            bi0, bi1, next0, next1);
3422         }
3423
3424       while (n_left_from > 0 && n_left_to_next > 0)
3425         {
3426           u32 bi0;
3427           vlib_buffer_t *b0;
3428           u32 next0, sw_if_index0, rx_fib_index0, iph_offset0 = 0, proto0,
3429               new_addr0, old_addr0;
3430           u16 old_port0, new_port0;
3431           ip4_header_t *ip0;
3432           udp_header_t *udp0;
3433           tcp_header_t *tcp0;
3434           icmp46_header_t * icmp0;
3435           snat_session_t *s0 = 0;
3436           clib_bihash_kv_16_8_t kv0, value0;
3437           ip_csum_t sum0;
3438
3439           /* speculatively enqueue b0 to the current next frame */
3440           bi0 = from[0];
3441           to_next[0] = bi0;
3442           from += 1;
3443           to_next += 1;
3444           n_left_from -= 1;
3445           n_left_to_next -= 1;
3446
3447           b0 = vlib_get_buffer (vm, bi0);
3448           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3449
3450           if (is_output_feature)
3451             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
3452
3453           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
3454                  iph_offset0);
3455
3456           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3457           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3458                                                                sw_if_index0);
3459
3460           if (PREDICT_FALSE(ip0->ttl == 1))
3461             {
3462               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3463               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3464                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3465                                            0);
3466               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3467               goto trace0;
3468             }
3469
3470           udp0 = ip4_next_header (ip0);
3471           tcp0 = (tcp_header_t *) udp0;
3472           icmp0 = (icmp46_header_t *) udp0;
3473           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3474
3475           if (is_slow_path)
3476             {
3477               if (PREDICT_FALSE (proto0 == ~0))
3478                 {
3479                   s0 = nat44_ed_in2out_unknown_proto (sm, b0, ip0,
3480                                                       rx_fib_index0,
3481                                                       thread_index, now, vm,
3482                                                       node);
3483                   if (!s0)
3484                     next0 = SNAT_IN2OUT_NEXT_DROP;
3485                   goto trace0;
3486                 }
3487
3488               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
3489                 {
3490                   next0 = icmp_in2out_slow_path
3491                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
3492                      next0, now, thread_index, &s0);
3493                   goto trace0;
3494                 }
3495             }
3496           else
3497             {
3498                if (is_output_feature)
3499                 {
3500                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(
3501                       sm, ip0, thread_index, now, vm, b0)))
3502                     goto trace0;
3503                 }
3504
3505               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
3506                 {
3507                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3508                   goto trace0;
3509                 }
3510
3511               if (ip4_is_fragment (ip0))
3512                 {
3513                   b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
3514                   next0 = SNAT_IN2OUT_NEXT_DROP;
3515                   goto trace0;
3516                 }
3517             }
3518
3519           make_ed_kv (&kv0, &ip0->src_address, &ip0->dst_address, ip0->protocol,
3520                       rx_fib_index0, udp0->src_port, udp0->dst_port);
3521
3522           if (clib_bihash_search_16_8 (&tsm->in2out_ed, &kv0, &value0))
3523             {
3524               if (is_slow_path)
3525                 {
3526                   if (is_output_feature)
3527                     {
3528                       if (PREDICT_FALSE(nat44_ed_not_translate_output_feature(
3529                           sm, ip0, ip0->protocol, udp0->src_port,
3530                           udp0->dst_port, thread_index, sw_if_index0)))
3531                         goto trace0;
3532                     }
3533                   else
3534                     {
3535                       if (PREDICT_FALSE(nat44_ed_not_translate(sm, node,
3536                           sw_if_index0, ip0, proto0, rx_fib_index0,
3537                           thread_index)))
3538                         goto trace0;
3539                     }
3540
3541                   next0 = slow_path_ed (sm, b0, rx_fib_index0, &kv0, &s0, node,
3542                                         next0, thread_index);
3543
3544                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
3545                     goto trace0;
3546                 }
3547               else
3548                 {
3549                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
3550                   goto trace0;
3551                 }
3552             }
3553           else
3554             {
3555               s0 = pool_elt_at_index (tsm->sessions, value0.value);
3556             }
3557
3558           b0->flags |= VNET_BUFFER_F_IS_NATED;
3559
3560           if (!is_output_feature)
3561             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
3562
3563           old_addr0 = ip0->src_address.as_u32;
3564           new_addr0 = ip0->src_address.as_u32 = s0->out2in.addr.as_u32;
3565           sum0 = ip0->checksum;
3566           sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3567                                  src_address);
3568           if (PREDICT_FALSE (is_twice_nat_session (s0)))
3569             sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3570                                    s0->ext_host_addr.as_u32, ip4_header_t,
3571                                    dst_address);
3572           ip0->checksum = ip_csum_fold (sum0);
3573
3574           if (PREDICT_TRUE (proto0 == SNAT_PROTOCOL_TCP))
3575             {
3576               old_port0 = tcp0->src_port;
3577               new_port0 = tcp0->src_port = s0->out2in.port;
3578
3579               sum0 = tcp0->checksum;
3580               sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
3581                                      dst_address);
3582               sum0 = ip_csum_update (sum0, old_port0, new_port0, ip4_header_t,
3583                                      length);
3584               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3585                 {
3586                   sum0 = ip_csum_update (sum0, ip0->dst_address.as_u32,
3587                                          s0->ext_host_addr.as_u32,
3588                                          ip4_header_t, dst_address);
3589                   sum0 = ip_csum_update (sum0, tcp0->dst_port,
3590                                          s0->ext_host_port, ip4_header_t,
3591                                          length);
3592                   tcp0->dst_port = s0->ext_host_port;
3593                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3594                 }
3595               tcp0->checksum = ip_csum_fold(sum0);
3596               if (nat44_set_tcp_session_state_i2o (sm, s0, tcp0, thread_index))
3597                 goto trace0;
3598             }
3599           else
3600             {
3601               udp0->src_port = s0->out2in.port;
3602               udp0->checksum = 0;
3603               if (PREDICT_FALSE (is_twice_nat_session (s0)))
3604                 {
3605                   udp0->dst_port = s0->ext_host_port;
3606                   ip0->dst_address.as_u32 = s0->ext_host_addr.as_u32;
3607                 }
3608             }
3609
3610           /* Accounting */
3611           nat44_session_update_counters (s0, now,
3612                                          vlib_buffer_length_in_chain (vm, b0));
3613           /* Per-user LRU list maintenance */
3614           nat44_session_update_lru (sm, s0, thread_index);
3615
3616         trace0:
3617           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3618                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3619             {
3620               snat_in2out_trace_t *t =
3621                 vlib_add_trace (vm, node, b0, sizeof (*t));
3622               t->is_slow_path = is_slow_path;
3623               t->sw_if_index = sw_if_index0;
3624               t->next_index = next0;
3625               t->session_index = ~0;
3626               if (s0)
3627                 t->session_index = s0 - tsm->sessions;
3628             }
3629
3630           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3631
3632           /* verify speculative enqueue, maybe switch current next frame */
3633           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3634                                            to_next, n_left_to_next,
3635                                            bi0, next0);
3636         }
3637
3638       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3639     }
3640
3641   vlib_node_increment_counter (vm, stats_node_index,
3642                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3643                                pkts_processed);
3644   return frame->n_vectors;
3645 }
3646
3647 static uword
3648 nat44_ed_in2out_fast_path_fn (vlib_main_t * vm,
3649                               vlib_node_runtime_t * node,
3650                               vlib_frame_t * frame)
3651 {
3652   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 0);
3653 }
3654
3655 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
3656   .function = nat44_ed_in2out_fast_path_fn,
3657   .name = "nat44-ed-in2out",
3658   .vector_size = sizeof (u32),
3659   .format_trace = format_snat_in2out_trace,
3660   .type = VLIB_NODE_TYPE_INTERNAL,
3661
3662   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3663   .error_strings = snat_in2out_error_strings,
3664
3665   .runtime_data_bytes = sizeof (snat_runtime_t),
3666
3667   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3668
3669   /* edit / add dispositions here */
3670   .next_nodes = {
3671     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3672     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3673     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3674     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3675     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3676   },
3677 };
3678
3679 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_node, nat44_ed_in2out_fast_path_fn);
3680
3681 static uword
3682 nat44_ed_in2out_output_fast_path_fn (vlib_main_t * vm,
3683                                      vlib_node_runtime_t * node,
3684                                      vlib_frame_t * frame)
3685 {
3686   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 0, 1);
3687 }
3688
3689 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
3690   .function = nat44_ed_in2out_output_fast_path_fn,
3691   .name = "nat44-ed-in2out-output",
3692   .vector_size = sizeof (u32),
3693   .format_trace = format_snat_in2out_trace,
3694   .type = VLIB_NODE_TYPE_INTERNAL,
3695
3696   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3697   .error_strings = snat_in2out_error_strings,
3698
3699   .runtime_data_bytes = sizeof (snat_runtime_t),
3700
3701   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3702
3703   /* edit / add dispositions here */
3704   .next_nodes = {
3705     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3706     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3707     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3708     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3709     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3710   },
3711 };
3712
3713 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_node,
3714                               nat44_ed_in2out_output_fast_path_fn);
3715
3716 static uword
3717 nat44_ed_in2out_slow_path_fn (vlib_main_t * vm,
3718                               vlib_node_runtime_t * node,
3719                               vlib_frame_t * frame)
3720 {
3721   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 0);
3722 }
3723
3724 VLIB_REGISTER_NODE (nat44_ed_in2out_slowpath_node) = {
3725   .function = nat44_ed_in2out_slow_path_fn,
3726   .name = "nat44-ed-in2out-slowpath",
3727   .vector_size = sizeof (u32),
3728   .format_trace = format_snat_in2out_trace,
3729   .type = VLIB_NODE_TYPE_INTERNAL,
3730
3731   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3732   .error_strings = snat_in2out_error_strings,
3733
3734   .runtime_data_bytes = sizeof (snat_runtime_t),
3735
3736   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3737
3738   /* edit / add dispositions here */
3739   .next_nodes = {
3740     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3741     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3742     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-slowpath",
3743     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3744     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3745   },
3746 };
3747
3748 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_slowpath_node,
3749                               nat44_ed_in2out_slow_path_fn);
3750
3751 static uword
3752 nat44_ed_in2out_output_slow_path_fn (vlib_main_t * vm,
3753                                      vlib_node_runtime_t * node,
3754                                      vlib_frame_t * frame)
3755 {
3756   return nat44_ed_in2out_node_fn_inline (vm, node, frame, 1, 1);
3757 }
3758
3759 VLIB_REGISTER_NODE (nat44_ed_in2out_output_slowpath_node) = {
3760   .function = nat44_ed_in2out_output_slow_path_fn,
3761   .name = "nat44-ed-in2out-output-slowpath",
3762   .vector_size = sizeof (u32),
3763   .format_trace = format_snat_in2out_trace,
3764   .type = VLIB_NODE_TYPE_INTERNAL,
3765
3766   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3767   .error_strings = snat_in2out_error_strings,
3768
3769   .runtime_data_bytes = sizeof (snat_runtime_t),
3770
3771   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
3772
3773   /* edit / add dispositions here */
3774   .next_nodes = {
3775     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3776     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
3777     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
3778     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3779     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
3780   },
3781 };
3782
3783 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_in2out_output_slowpath_node,
3784                               nat44_ed_in2out_output_slow_path_fn);
3785
3786 /**************************/
3787 /*** deterministic mode ***/
3788 /**************************/
3789 static uword
3790 snat_det_in2out_node_fn (vlib_main_t * vm,
3791                          vlib_node_runtime_t * node,
3792                          vlib_frame_t * frame)
3793 {
3794   u32 n_left_from, * from, * to_next;
3795   snat_in2out_next_t next_index;
3796   u32 pkts_processed = 0;
3797   snat_main_t * sm = &snat_main;
3798   u32 now = (u32) vlib_time_now (vm);
3799   u32 thread_index = vm->thread_index;
3800
3801   from = vlib_frame_vector_args (frame);
3802   n_left_from = frame->n_vectors;
3803   next_index = node->cached_next_index;
3804
3805   while (n_left_from > 0)
3806     {
3807       u32 n_left_to_next;
3808
3809       vlib_get_next_frame (vm, node, next_index,
3810                            to_next, n_left_to_next);
3811
3812       while (n_left_from >= 4 && n_left_to_next >= 2)
3813         {
3814           u32 bi0, bi1;
3815           vlib_buffer_t * b0, * b1;
3816           u32 next0, next1;
3817           u32 sw_if_index0, sw_if_index1;
3818           ip4_header_t * ip0, * ip1;
3819           ip_csum_t sum0, sum1;
3820           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
3821           u16 old_port0, new_port0, lo_port0, i0;
3822           u16 old_port1, new_port1, lo_port1, i1;
3823           udp_header_t * udp0, * udp1;
3824           tcp_header_t * tcp0, * tcp1;
3825           u32 proto0, proto1;
3826           snat_det_out_key_t key0, key1;
3827           snat_det_map_t * dm0, * dm1;
3828           snat_det_session_t * ses0 = 0, * ses1 = 0;
3829           u32 rx_fib_index0, rx_fib_index1;
3830           icmp46_header_t * icmp0, * icmp1;
3831
3832           /* Prefetch next iteration. */
3833           {
3834             vlib_buffer_t * p2, * p3;
3835
3836             p2 = vlib_get_buffer (vm, from[2]);
3837             p3 = vlib_get_buffer (vm, from[3]);
3838
3839             vlib_prefetch_buffer_header (p2, LOAD);
3840             vlib_prefetch_buffer_header (p3, LOAD);
3841
3842             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
3843             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
3844           }
3845
3846           /* speculatively enqueue b0 and b1 to the current next frame */
3847           to_next[0] = bi0 = from[0];
3848           to_next[1] = bi1 = from[1];
3849           from += 2;
3850           to_next += 2;
3851           n_left_from -= 2;
3852           n_left_to_next -= 2;
3853
3854           b0 = vlib_get_buffer (vm, bi0);
3855           b1 = vlib_get_buffer (vm, bi1);
3856
3857           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3858           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
3859
3860           ip0 = vlib_buffer_get_current (b0);
3861           udp0 = ip4_next_header (ip0);
3862           tcp0 = (tcp_header_t *) udp0;
3863
3864           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3865
3866           if (PREDICT_FALSE(ip0->ttl == 1))
3867             {
3868               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3869               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3870                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3871                                            0);
3872               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3873               goto trace0;
3874             }
3875
3876           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3877
3878           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3879             {
3880               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3881               icmp0 = (icmp46_header_t *) udp0;
3882
3883               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3884                                   rx_fib_index0, node, next0, thread_index,
3885                                   &ses0, &dm0);
3886               goto trace0;
3887             }
3888
3889           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3890           if (PREDICT_FALSE(!dm0))
3891             {
3892               nat_log_info ("no match for internal host %U",
3893                             format_ip4_address, &ip0->src_address);
3894               next0 = SNAT_IN2OUT_NEXT_DROP;
3895               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3896               goto trace0;
3897             }
3898
3899           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3900
3901           key0.ext_host_addr = ip0->dst_address;
3902           key0.ext_host_port = tcp0->dst;
3903
3904           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3905           if (PREDICT_FALSE(!ses0))
3906             {
3907               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3908                 {
3909                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3910                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3911
3912                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3913                     continue;
3914
3915                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3916                   break;
3917                 }
3918               if (PREDICT_FALSE(!ses0))
3919                 {
3920                   /* too many sessions for user, send ICMP error packet */
3921
3922                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3923                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3924                                                ICMP4_destination_unreachable_destination_unreachable_host,
3925                                                0);
3926                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3927                   goto trace0;
3928                 }
3929             }
3930
3931           new_port0 = ses0->out.out_port;
3932
3933           old_addr0.as_u32 = ip0->src_address.as_u32;
3934           ip0->src_address.as_u32 = new_addr0.as_u32;
3935           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3936
3937           sum0 = ip0->checksum;
3938           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3939                                  ip4_header_t,
3940                                  src_address /* changed member */);
3941           ip0->checksum = ip_csum_fold (sum0);
3942
3943           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3944             {
3945               if (tcp0->flags & TCP_FLAG_SYN)
3946                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3947               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3948                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3949               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3950                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3951               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3952                 snat_det_ses_close(dm0, ses0);
3953               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3954                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3955               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3956                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3957
3958               old_port0 = tcp0->src;
3959               tcp0->src = new_port0;
3960
3961               sum0 = tcp0->checksum;
3962               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3963                                      ip4_header_t,
3964                                      dst_address /* changed member */);
3965               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3966                                      ip4_header_t /* cheat */,
3967                                      length /* changed member */);
3968               tcp0->checksum = ip_csum_fold(sum0);
3969             }
3970           else
3971             {
3972               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3973               old_port0 = udp0->src_port;
3974               udp0->src_port = new_port0;
3975               udp0->checksum = 0;
3976             }
3977
3978           switch(ses0->state)
3979             {
3980             case SNAT_SESSION_UDP_ACTIVE:
3981                 ses0->expire = now + sm->udp_timeout;
3982                 break;
3983             case SNAT_SESSION_TCP_SYN_SENT:
3984             case SNAT_SESSION_TCP_FIN_WAIT:
3985             case SNAT_SESSION_TCP_CLOSE_WAIT:
3986             case SNAT_SESSION_TCP_LAST_ACK:
3987                 ses0->expire = now + sm->tcp_transitory_timeout;
3988                 break;
3989             case SNAT_SESSION_TCP_ESTABLISHED:
3990                 ses0->expire = now + sm->tcp_established_timeout;
3991                 break;
3992             }
3993
3994         trace0:
3995           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3996                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3997             {
3998               snat_in2out_trace_t *t =
3999                  vlib_add_trace (vm, node, b0, sizeof (*t));
4000               t->is_slow_path = 0;
4001               t->sw_if_index = sw_if_index0;
4002               t->next_index = next0;
4003               t->session_index = ~0;
4004               if (ses0)
4005                 t->session_index = ses0 - dm0->sessions;
4006             }
4007
4008           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4009
4010           ip1 = vlib_buffer_get_current (b1);
4011           udp1 = ip4_next_header (ip1);
4012           tcp1 = (tcp_header_t *) udp1;
4013
4014           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
4015
4016           if (PREDICT_FALSE(ip1->ttl == 1))
4017             {
4018               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4019               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
4020                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4021                                            0);
4022               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4023               goto trace1;
4024             }
4025
4026           proto1 = ip_proto_to_snat_proto (ip1->protocol);
4027
4028           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
4029             {
4030               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
4031               icmp1 = (icmp46_header_t *) udp1;
4032
4033               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
4034                                   rx_fib_index1, node, next1, thread_index,
4035                                   &ses1, &dm1);
4036               goto trace1;
4037             }
4038
4039           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
4040           if (PREDICT_FALSE(!dm1))
4041             {
4042               nat_log_info ("no match for internal host %U",
4043                             format_ip4_address, &ip0->src_address);
4044               next1 = SNAT_IN2OUT_NEXT_DROP;
4045               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4046               goto trace1;
4047             }
4048
4049           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
4050
4051           key1.ext_host_addr = ip1->dst_address;
4052           key1.ext_host_port = tcp1->dst;
4053
4054           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
4055           if (PREDICT_FALSE(!ses1))
4056             {
4057               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
4058                 {
4059                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
4060                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
4061
4062                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
4063                     continue;
4064
4065                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
4066                   break;
4067                 }
4068               if (PREDICT_FALSE(!ses1))
4069                 {
4070                   /* too many sessions for user, send ICMP error packet */
4071
4072                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4073                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
4074                                                ICMP4_destination_unreachable_destination_unreachable_host,
4075                                                0);
4076                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4077                   goto trace1;
4078                 }
4079             }
4080
4081           new_port1 = ses1->out.out_port;
4082
4083           old_addr1.as_u32 = ip1->src_address.as_u32;
4084           ip1->src_address.as_u32 = new_addr1.as_u32;
4085           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4086
4087           sum1 = ip1->checksum;
4088           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4089                                  ip4_header_t,
4090                                  src_address /* changed member */);
4091           ip1->checksum = ip_csum_fold (sum1);
4092
4093           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
4094             {
4095               if (tcp1->flags & TCP_FLAG_SYN)
4096                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
4097               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
4098                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4099               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
4100                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
4101               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
4102                 snat_det_ses_close(dm1, ses1);
4103               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4104                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
4105               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
4106                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
4107
4108               old_port1 = tcp1->src;
4109               tcp1->src = new_port1;
4110
4111               sum1 = tcp1->checksum;
4112               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
4113                                      ip4_header_t,
4114                                      dst_address /* changed member */);
4115               sum1 = ip_csum_update (sum1, old_port1, new_port1,
4116                                      ip4_header_t /* cheat */,
4117                                      length /* changed member */);
4118               tcp1->checksum = ip_csum_fold(sum1);
4119             }
4120           else
4121             {
4122               ses1->state = SNAT_SESSION_UDP_ACTIVE;
4123               old_port1 = udp1->src_port;
4124               udp1->src_port = new_port1;
4125               udp1->checksum = 0;
4126             }
4127
4128           switch(ses1->state)
4129             {
4130             case SNAT_SESSION_UDP_ACTIVE:
4131                 ses1->expire = now + sm->udp_timeout;
4132                 break;
4133             case SNAT_SESSION_TCP_SYN_SENT:
4134             case SNAT_SESSION_TCP_FIN_WAIT:
4135             case SNAT_SESSION_TCP_CLOSE_WAIT:
4136             case SNAT_SESSION_TCP_LAST_ACK:
4137                 ses1->expire = now + sm->tcp_transitory_timeout;
4138                 break;
4139             case SNAT_SESSION_TCP_ESTABLISHED:
4140                 ses1->expire = now + sm->tcp_established_timeout;
4141                 break;
4142             }
4143
4144         trace1:
4145           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4146                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
4147             {
4148               snat_in2out_trace_t *t =
4149                  vlib_add_trace (vm, node, b1, sizeof (*t));
4150               t->is_slow_path = 0;
4151               t->sw_if_index = sw_if_index1;
4152               t->next_index = next1;
4153               t->session_index = ~0;
4154               if (ses1)
4155                 t->session_index = ses1 - dm1->sessions;
4156             }
4157
4158           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
4159
4160           /* verify speculative enqueues, maybe switch current next frame */
4161           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
4162                                            to_next, n_left_to_next,
4163                                            bi0, bi1, next0, next1);
4164          }
4165
4166       while (n_left_from > 0 && n_left_to_next > 0)
4167         {
4168           u32 bi0;
4169           vlib_buffer_t * b0;
4170           u32 next0;
4171           u32 sw_if_index0;
4172           ip4_header_t * ip0;
4173           ip_csum_t sum0;
4174           ip4_address_t new_addr0, old_addr0;
4175           u16 old_port0, new_port0, lo_port0, i0;
4176           udp_header_t * udp0;
4177           tcp_header_t * tcp0;
4178           u32 proto0;
4179           snat_det_out_key_t key0;
4180           snat_det_map_t * dm0;
4181           snat_det_session_t * ses0 = 0;
4182           u32 rx_fib_index0;
4183           icmp46_header_t * icmp0;
4184
4185           /* speculatively enqueue b0 to the current next frame */
4186           bi0 = from[0];
4187           to_next[0] = bi0;
4188           from += 1;
4189           to_next += 1;
4190           n_left_from -= 1;
4191           n_left_to_next -= 1;
4192
4193           b0 = vlib_get_buffer (vm, bi0);
4194           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4195
4196           ip0 = vlib_buffer_get_current (b0);
4197           udp0 = ip4_next_header (ip0);
4198           tcp0 = (tcp_header_t *) udp0;
4199
4200           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4201
4202           if (PREDICT_FALSE(ip0->ttl == 1))
4203             {
4204               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4205               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4206                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4207                                            0);
4208               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4209               goto trace00;
4210             }
4211
4212           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4213
4214           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
4215             {
4216               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4217               icmp0 = (icmp46_header_t *) udp0;
4218
4219               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4220                                   rx_fib_index0, node, next0, thread_index,
4221                                   &ses0, &dm0);
4222               goto trace00;
4223             }
4224
4225           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
4226           if (PREDICT_FALSE(!dm0))
4227             {
4228               nat_log_info ("no match for internal host %U",
4229                             format_ip4_address, &ip0->src_address);
4230               next0 = SNAT_IN2OUT_NEXT_DROP;
4231               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4232               goto trace00;
4233             }
4234
4235           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
4236
4237           key0.ext_host_addr = ip0->dst_address;
4238           key0.ext_host_port = tcp0->dst;
4239
4240           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
4241           if (PREDICT_FALSE(!ses0))
4242             {
4243               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4244                 {
4245                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
4246                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
4247
4248                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
4249                     continue;
4250
4251                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
4252                   break;
4253                 }
4254               if (PREDICT_FALSE(!ses0))
4255                 {
4256                   /* too many sessions for user, send ICMP error packet */
4257
4258                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4259                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
4260                                                ICMP4_destination_unreachable_destination_unreachable_host,
4261                                                0);
4262                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4263                   goto trace00;
4264                 }
4265             }
4266
4267           new_port0 = ses0->out.out_port;
4268
4269           old_addr0.as_u32 = ip0->src_address.as_u32;
4270           ip0->src_address.as_u32 = new_addr0.as_u32;
4271           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
4272
4273           sum0 = ip0->checksum;
4274           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4275                                  ip4_header_t,
4276                                  src_address /* changed member */);
4277           ip0->checksum = ip_csum_fold (sum0);
4278
4279           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4280             {
4281               if (tcp0->flags & TCP_FLAG_SYN)
4282                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
4283               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
4284                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4285               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
4286                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
4287               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
4288                 snat_det_ses_close(dm0, ses0);
4289               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
4290                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
4291               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
4292                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
4293
4294               old_port0 = tcp0->src;
4295               tcp0->src = new_port0;
4296
4297               sum0 = tcp0->checksum;
4298               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
4299                                      ip4_header_t,
4300                                      dst_address /* changed member */);
4301               sum0 = ip_csum_update (sum0, old_port0, new_port0,
4302                                      ip4_header_t /* cheat */,
4303                                      length /* changed member */);
4304               tcp0->checksum = ip_csum_fold(sum0);
4305             }
4306           else
4307             {
4308               ses0->state = SNAT_SESSION_UDP_ACTIVE;
4309               old_port0 = udp0->src_port;
4310               udp0->src_port = new_port0;
4311               udp0->checksum = 0;
4312             }
4313
4314           switch(ses0->state)
4315             {
4316             case SNAT_SESSION_UDP_ACTIVE:
4317                 ses0->expire = now + sm->udp_timeout;
4318                 break;
4319             case SNAT_SESSION_TCP_SYN_SENT:
4320             case SNAT_SESSION_TCP_FIN_WAIT:
4321             case SNAT_SESSION_TCP_CLOSE_WAIT:
4322             case SNAT_SESSION_TCP_LAST_ACK:
4323                 ses0->expire = now + sm->tcp_transitory_timeout;
4324                 break;
4325             case SNAT_SESSION_TCP_ESTABLISHED:
4326                 ses0->expire = now + sm->tcp_established_timeout;
4327                 break;
4328             }
4329
4330         trace00:
4331           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4332                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4333             {
4334               snat_in2out_trace_t *t =
4335                  vlib_add_trace (vm, node, b0, sizeof (*t));
4336               t->is_slow_path = 0;
4337               t->sw_if_index = sw_if_index0;
4338               t->next_index = next0;
4339               t->session_index = ~0;
4340               if (ses0)
4341                 t->session_index = ses0 - dm0->sessions;
4342             }
4343
4344           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4345
4346           /* verify speculative enqueue, maybe switch current next frame */
4347           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4348                                            to_next, n_left_to_next,
4349                                            bi0, next0);
4350         }
4351
4352       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4353     }
4354
4355   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
4356                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4357                                pkts_processed);
4358   return frame->n_vectors;
4359 }
4360
4361 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
4362   .function = snat_det_in2out_node_fn,
4363   .name = "nat44-det-in2out",
4364   .vector_size = sizeof (u32),
4365   .format_trace = format_snat_in2out_trace,
4366   .type = VLIB_NODE_TYPE_INTERNAL,
4367
4368   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4369   .error_strings = snat_in2out_error_strings,
4370
4371   .runtime_data_bytes = sizeof (snat_runtime_t),
4372
4373   .n_next_nodes = 3,
4374
4375   /* edit / add dispositions here */
4376   .next_nodes = {
4377     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4378     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4379     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4380   },
4381 };
4382
4383 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
4384
4385 /**
4386  * Get address and port values to be used for ICMP packet translation
4387  * and create session if needed
4388  *
4389  * @param[in,out] sm             NAT main
4390  * @param[in,out] node           NAT node runtime
4391  * @param[in] thread_index       thread index
4392  * @param[in,out] b0             buffer containing packet to be translated
4393  * @param[out] p_proto           protocol used for matching
4394  * @param[out] p_value           address and port after NAT translation
4395  * @param[out] p_dont_translate  if packet should not be translated
4396  * @param d                      optional parameter
4397  * @param e                      optional parameter
4398  */
4399 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
4400                           u32 thread_index, vlib_buffer_t *b0,
4401                           ip4_header_t *ip0, u8 *p_proto,
4402                           snat_session_key_t *p_value,
4403                           u8 *p_dont_translate, void *d, void *e)
4404 {
4405   icmp46_header_t *icmp0;
4406   u32 sw_if_index0;
4407   u32 rx_fib_index0;
4408   u8 protocol;
4409   snat_det_out_key_t key0;
4410   u8 dont_translate = 0;
4411   u32 next0 = ~0;
4412   icmp_echo_header_t *echo0, *inner_echo0 = 0;
4413   ip4_header_t *inner_ip0;
4414   void *l4_header = 0;
4415   icmp46_header_t *inner_icmp0;
4416   snat_det_map_t * dm0 = 0;
4417   ip4_address_t new_addr0;
4418   u16 lo_port0, i0;
4419   snat_det_session_t * ses0 = 0;
4420   ip4_address_t in_addr;
4421   u16 in_port;
4422
4423   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
4424   echo0 = (icmp_echo_header_t *)(icmp0+1);
4425   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4426   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
4427
4428   if (!icmp_is_error_message (icmp0))
4429     {
4430       protocol = SNAT_PROTOCOL_ICMP;
4431       in_addr = ip0->src_address;
4432       in_port = echo0->identifier;
4433     }
4434   else
4435     {
4436       inner_ip0 = (ip4_header_t *)(echo0+1);
4437       l4_header = ip4_next_header (inner_ip0);
4438       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
4439       in_addr = inner_ip0->dst_address;
4440       switch (protocol)
4441         {
4442         case SNAT_PROTOCOL_ICMP:
4443           inner_icmp0 = (icmp46_header_t*)l4_header;
4444           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
4445           in_port = inner_echo0->identifier;
4446           break;
4447         case SNAT_PROTOCOL_UDP:
4448         case SNAT_PROTOCOL_TCP:
4449           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
4450           break;
4451         default:
4452           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
4453           next0 = SNAT_IN2OUT_NEXT_DROP;
4454           goto out;
4455         }
4456     }
4457
4458   dm0 = snat_det_map_by_user(sm, &in_addr);
4459   if (PREDICT_FALSE(!dm0))
4460     {
4461       nat_log_info ("no match for internal host %U",
4462                     format_ip4_address, &in_addr);
4463       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4464           IP_PROTOCOL_ICMP, rx_fib_index0)))
4465         {
4466           dont_translate = 1;
4467           goto out;
4468         }
4469       next0 = SNAT_IN2OUT_NEXT_DROP;
4470       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4471       goto out;
4472     }
4473
4474   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
4475
4476   key0.ext_host_addr = ip0->dst_address;
4477   key0.ext_host_port = 0;
4478
4479   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
4480   if (PREDICT_FALSE(!ses0))
4481     {
4482       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
4483           IP_PROTOCOL_ICMP, rx_fib_index0)))
4484         {
4485           dont_translate = 1;
4486           goto out;
4487         }
4488       if (icmp0->type != ICMP4_echo_request)
4489         {
4490           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4491           next0 = SNAT_IN2OUT_NEXT_DROP;
4492           goto out;
4493         }
4494       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
4495         {
4496           key0.out_port = clib_host_to_net_u16 (lo_port0 +
4497             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
4498
4499           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
4500             continue;
4501
4502           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
4503           break;
4504         }
4505       if (PREDICT_FALSE(!ses0))
4506         {
4507           next0 = SNAT_IN2OUT_NEXT_DROP;
4508           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
4509           goto out;
4510         }
4511     }
4512
4513   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
4514                     !icmp_is_error_message (icmp0)))
4515     {
4516       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
4517       next0 = SNAT_IN2OUT_NEXT_DROP;
4518       goto out;
4519     }
4520
4521   u32 now = (u32) vlib_time_now (sm->vlib_main);
4522
4523   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
4524   ses0->expire = now + sm->icmp_timeout;
4525
4526 out:
4527   *p_proto = protocol;
4528   if (ses0)
4529     {
4530       p_value->addr = new_addr0;
4531       p_value->fib_index = sm->outside_fib_index;
4532       p_value->port = ses0->out.out_port;
4533     }
4534   *p_dont_translate = dont_translate;
4535   if (d)
4536     *(snat_det_session_t**)d = ses0;
4537   if (e)
4538     *(snat_det_map_t**)e = dm0;
4539   return next0;
4540 }
4541
4542 /**********************/
4543 /*** worker handoff ***/
4544 /**********************/
4545 static inline uword
4546 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
4547                                       vlib_node_runtime_t * node,
4548                                       vlib_frame_t * frame,
4549                                       u8 is_output)
4550 {
4551   snat_main_t *sm = &snat_main;
4552   vlib_thread_main_t *tm = vlib_get_thread_main ();
4553   u32 n_left_from, *from, *to_next = 0, *to_next_drop = 0;
4554   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
4555   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
4556     = 0;
4557   vlib_frame_queue_elt_t *hf = 0;
4558   vlib_frame_queue_t *fq;
4559   vlib_frame_t *f = 0;
4560   int i;
4561   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
4562   u32 next_worker_index = 0;
4563   u32 current_worker_index = ~0;
4564   u32 thread_index = vm->thread_index;
4565   u32 fq_index;
4566   u32 to_node_index;
4567   vlib_frame_t *d = 0;
4568
4569   ASSERT (vec_len (sm->workers));
4570
4571   if (is_output)
4572     {
4573       fq_index = sm->fq_in2out_output_index;
4574       to_node_index = sm->in2out_output_node_index;
4575     }
4576   else
4577     {
4578       fq_index = sm->fq_in2out_index;
4579       to_node_index = sm->in2out_node_index;
4580     }
4581
4582   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
4583     {
4584       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
4585
4586       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
4587                                tm->n_vlib_mains - 1,
4588                                (vlib_frame_queue_t *) (~0));
4589     }
4590
4591   from = vlib_frame_vector_args (frame);
4592   n_left_from = frame->n_vectors;
4593
4594   while (n_left_from > 0)
4595     {
4596       u32 bi0;
4597       vlib_buffer_t *b0;
4598       u32 sw_if_index0;
4599       u32 rx_fib_index0;
4600       ip4_header_t * ip0;
4601       u8 do_handoff;
4602
4603       bi0 = from[0];
4604       from += 1;
4605       n_left_from -= 1;
4606
4607       b0 = vlib_get_buffer (vm, bi0);
4608
4609       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
4610       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4611
4612       ip0 = vlib_buffer_get_current (b0);
4613
4614       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
4615
4616       if (PREDICT_FALSE (next_worker_index != thread_index))
4617         {
4618           do_handoff = 1;
4619
4620           if (next_worker_index != current_worker_index)
4621             {
4622               fq = is_vlib_frame_queue_congested (
4623                 fq_index, next_worker_index, NAT_FQ_NELTS - 2,
4624                 congested_handoff_queue_by_worker_index);
4625
4626               if (fq)
4627                 {
4628                   /* if this is 1st frame */
4629                   if (!d)
4630                     {
4631                       d = vlib_get_frame_to_node (vm, sm->error_node_index);
4632                       to_next_drop = vlib_frame_vector_args (d);
4633                     }
4634
4635                   to_next_drop[0] = bi0;
4636                   to_next_drop += 1;
4637                   d->n_vectors++;
4638                   b0->error = node->errors[SNAT_IN2OUT_ERROR_FQ_CONGESTED];
4639                   goto trace0;
4640                 }
4641
4642               if (hf)
4643                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4644
4645               hf = vlib_get_worker_handoff_queue_elt (fq_index,
4646                                                       next_worker_index,
4647                                                       handoff_queue_elt_by_worker_index);
4648
4649               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
4650               to_next_worker = &hf->buffer_index[hf->n_vectors];
4651               current_worker_index = next_worker_index;
4652             }
4653
4654           /* enqueue to correct worker thread */
4655           to_next_worker[0] = bi0;
4656           to_next_worker++;
4657           n_left_to_next_worker--;
4658
4659           if (n_left_to_next_worker == 0)
4660             {
4661               hf->n_vectors = VLIB_FRAME_SIZE;
4662               vlib_put_frame_queue_elt (hf);
4663               current_worker_index = ~0;
4664               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
4665               hf = 0;
4666             }
4667         }
4668       else
4669         {
4670           do_handoff = 0;
4671           /* if this is 1st frame */
4672           if (!f)
4673             {
4674               f = vlib_get_frame_to_node (vm, to_node_index);
4675               to_next = vlib_frame_vector_args (f);
4676             }
4677
4678           to_next[0] = bi0;
4679           to_next += 1;
4680           f->n_vectors++;
4681         }
4682
4683 trace0:
4684       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
4685                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4686         {
4687           snat_in2out_worker_handoff_trace_t *t =
4688             vlib_add_trace (vm, node, b0, sizeof (*t));
4689           t->next_worker_index = next_worker_index;
4690           t->do_handoff = do_handoff;
4691         }
4692     }
4693
4694   if (f)
4695     vlib_put_frame_to_node (vm, to_node_index, f);
4696
4697   if (d)
4698     vlib_put_frame_to_node (vm, sm->error_node_index, d);
4699
4700   if (hf)
4701     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
4702
4703   /* Ship frames to the worker nodes */
4704   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
4705     {
4706       if (handoff_queue_elt_by_worker_index[i])
4707         {
4708           hf = handoff_queue_elt_by_worker_index[i];
4709           /*
4710            * It works better to let the handoff node
4711            * rate-adapt, always ship the handoff queue element.
4712            */
4713           if (1 || hf->n_vectors == hf->last_n_vectors)
4714             {
4715               vlib_put_frame_queue_elt (hf);
4716               handoff_queue_elt_by_worker_index[i] = 0;
4717             }
4718           else
4719             hf->last_n_vectors = hf->n_vectors;
4720         }
4721       congested_handoff_queue_by_worker_index[i] =
4722         (vlib_frame_queue_t *) (~0);
4723     }
4724   hf = 0;
4725   current_worker_index = ~0;
4726   return frame->n_vectors;
4727 }
4728
4729 static uword
4730 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
4731                                vlib_node_runtime_t * node,
4732                                vlib_frame_t * frame)
4733 {
4734   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
4735 }
4736
4737 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
4738   .function = snat_in2out_worker_handoff_fn,
4739   .name = "nat44-in2out-worker-handoff",
4740   .vector_size = sizeof (u32),
4741   .format_trace = format_snat_in2out_worker_handoff_trace,
4742   .type = VLIB_NODE_TYPE_INTERNAL,
4743
4744   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4745   .error_strings = snat_in2out_error_strings,
4746
4747   .n_next_nodes = 1,
4748
4749   .next_nodes = {
4750     [0] = "error-drop",
4751   },
4752 };
4753
4754 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
4755                               snat_in2out_worker_handoff_fn);
4756
4757 static uword
4758 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
4759                                       vlib_node_runtime_t * node,
4760                                       vlib_frame_t * frame)
4761 {
4762   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
4763 }
4764
4765 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
4766   .function = snat_in2out_output_worker_handoff_fn,
4767   .name = "nat44-in2out-output-worker-handoff",
4768   .vector_size = sizeof (u32),
4769   .format_trace = format_snat_in2out_worker_handoff_trace,
4770   .type = VLIB_NODE_TYPE_INTERNAL,
4771
4772   .n_next_nodes = 1,
4773
4774   .next_nodes = {
4775     [0] = "error-drop",
4776   },
4777 };
4778
4779 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
4780                               snat_in2out_output_worker_handoff_fn);
4781
4782 static_always_inline int
4783 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
4784 {
4785   snat_address_t * ap;
4786   clib_bihash_kv_8_8_t kv, value;
4787   snat_session_key_t m_key;
4788
4789   vec_foreach (ap, sm->addresses)
4790     {
4791       if (ap->addr.as_u32 == dst_addr->as_u32)
4792         return 1;
4793     }
4794
4795   m_key.addr.as_u32 = dst_addr->as_u32;
4796   m_key.fib_index = 0;
4797   m_key.port = 0;
4798   m_key.protocol = 0;
4799   kv.key = m_key.as_u64;
4800   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
4801     return 1;
4802
4803   return 0;
4804 }
4805
4806 static inline uword
4807 snat_hairpin_dst_fn_inline (vlib_main_t * vm,
4808                             vlib_node_runtime_t * node,
4809                             vlib_frame_t * frame,
4810                             int is_ed)
4811 {
4812   u32 n_left_from, * from, * to_next, stats_node_index;
4813   snat_in2out_next_t next_index;
4814   u32 pkts_processed = 0;
4815   snat_main_t * sm = &snat_main;
4816
4817   stats_node_index = is_ed ? nat44_ed_hairpin_dst_node.index :
4818     snat_hairpin_dst_node.index;
4819
4820   from = vlib_frame_vector_args (frame);
4821   n_left_from = frame->n_vectors;
4822   next_index = node->cached_next_index;
4823
4824   while (n_left_from > 0)
4825     {
4826       u32 n_left_to_next;
4827
4828       vlib_get_next_frame (vm, node, next_index,
4829                            to_next, n_left_to_next);
4830
4831       while (n_left_from > 0 && n_left_to_next > 0)
4832         {
4833           u32 bi0;
4834           vlib_buffer_t * b0;
4835           u32 next0;
4836           ip4_header_t * ip0;
4837           u32 proto0;
4838
4839           /* speculatively enqueue b0 to the current next frame */
4840           bi0 = from[0];
4841           to_next[0] = bi0;
4842           from += 1;
4843           to_next += 1;
4844           n_left_from -= 1;
4845           n_left_to_next -= 1;
4846
4847           b0 = vlib_get_buffer (vm, bi0);
4848           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4849           ip0 = vlib_buffer_get_current (b0);
4850
4851           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4852
4853           vnet_buffer (b0)->snat.flags = 0;
4854           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
4855             {
4856               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
4857                 {
4858                   udp_header_t * udp0 = ip4_next_header (ip0);
4859                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
4860
4861                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, is_ed);
4862                 }
4863               else if (proto0 == SNAT_PROTOCOL_ICMP)
4864                 {
4865                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
4866
4867                   snat_icmp_hairpinning (sm, b0, ip0, icmp0, is_ed);
4868                 }
4869               else
4870                 {
4871                   if (is_ed)
4872                     nat44_ed_hairpinning_unknown_proto (sm, b0, ip0);
4873                   else
4874                     nat_hairpinning_sm_unknown_proto (sm, b0, ip0);
4875                 }
4876
4877               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
4878             }
4879
4880           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4881
4882           /* verify speculative enqueue, maybe switch current next frame */
4883           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4884                                            to_next, n_left_to_next,
4885                                            bi0, next0);
4886          }
4887
4888       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4889     }
4890
4891   vlib_node_increment_counter (vm, stats_node_index,
4892                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4893                                pkts_processed);
4894   return frame->n_vectors;
4895 }
4896
4897 static uword
4898 snat_hairpin_dst_fn (vlib_main_t * vm,
4899                      vlib_node_runtime_t * node,
4900                      vlib_frame_t * frame)
4901 {
4902   return snat_hairpin_dst_fn_inline (vm, node, frame, 0);
4903 }
4904
4905 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
4906   .function = snat_hairpin_dst_fn,
4907   .name = "nat44-hairpin-dst",
4908   .vector_size = sizeof (u32),
4909   .type = VLIB_NODE_TYPE_INTERNAL,
4910   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4911   .error_strings = snat_in2out_error_strings,
4912   .n_next_nodes = 2,
4913   .next_nodes = {
4914     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4915     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4916   },
4917 };
4918
4919 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
4920                               snat_hairpin_dst_fn);
4921
4922 static uword
4923 nat44_ed_hairpin_dst_fn (vlib_main_t * vm,
4924                          vlib_node_runtime_t * node,
4925                          vlib_frame_t * frame)
4926 {
4927   return snat_hairpin_dst_fn_inline (vm, node, frame, 1);
4928 }
4929
4930 VLIB_REGISTER_NODE (nat44_ed_hairpin_dst_node) = {
4931   .function = nat44_ed_hairpin_dst_fn,
4932   .name = "nat44-ed-hairpin-dst",
4933   .vector_size = sizeof (u32),
4934   .type = VLIB_NODE_TYPE_INTERNAL,
4935   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4936   .error_strings = snat_in2out_error_strings,
4937   .n_next_nodes = 2,
4938   .next_nodes = {
4939     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4940     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4941   },
4942 };
4943
4944 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_dst_node,
4945                               nat44_ed_hairpin_dst_fn);
4946
4947 static inline uword
4948 snat_hairpin_src_fn_inline (vlib_main_t * vm,
4949                             vlib_node_runtime_t * node,
4950                             vlib_frame_t * frame,
4951                             int is_ed)
4952 {
4953   u32 n_left_from, * from, * to_next, stats_node_index;
4954   snat_in2out_next_t next_index;
4955   u32 pkts_processed = 0;
4956   snat_main_t *sm = &snat_main;
4957
4958   stats_node_index = is_ed ? nat44_ed_hairpin_src_node.index :
4959     snat_hairpin_src_node.index;
4960
4961   from = vlib_frame_vector_args (frame);
4962   n_left_from = frame->n_vectors;
4963   next_index = node->cached_next_index;
4964
4965   while (n_left_from > 0)
4966     {
4967       u32 n_left_to_next;
4968
4969       vlib_get_next_frame (vm, node, next_index,
4970                            to_next, n_left_to_next);
4971
4972       while (n_left_from > 0 && n_left_to_next > 0)
4973         {
4974           u32 bi0;
4975           vlib_buffer_t * b0;
4976           u32 next0;
4977           snat_interface_t *i;
4978           u32 sw_if_index0;
4979
4980           /* speculatively enqueue b0 to the current next frame */
4981           bi0 = from[0];
4982           to_next[0] = bi0;
4983           from += 1;
4984           to_next += 1;
4985           n_left_from -= 1;
4986           n_left_to_next -= 1;
4987
4988           b0 = vlib_get_buffer (vm, bi0);
4989           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4990           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
4991
4992           pool_foreach (i, sm->output_feature_interfaces,
4993           ({
4994             /* Only packets from NAT inside interface */
4995             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
4996               {
4997                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
4998                                     SNAT_FLAG_HAIRPINNING))
4999                   {
5000                     if (PREDICT_TRUE (sm->num_workers > 1))
5001                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
5002                     else
5003                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
5004                   }
5005                 break;
5006               }
5007           }));
5008
5009           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5010
5011           /* verify speculative enqueue, maybe switch current next frame */
5012           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5013                                            to_next, n_left_to_next,
5014                                            bi0, next0);
5015          }
5016
5017       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5018     }
5019
5020   vlib_node_increment_counter (vm, stats_node_index,
5021                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5022                                pkts_processed);
5023   return frame->n_vectors;
5024 }
5025
5026 static uword
5027 snat_hairpin_src_fn (vlib_main_t * vm,
5028                      vlib_node_runtime_t * node,
5029                      vlib_frame_t * frame)
5030 {
5031   return snat_hairpin_src_fn_inline (vm, node, frame, 0);
5032 }
5033
5034 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
5035   .function = snat_hairpin_src_fn,
5036   .name = "nat44-hairpin-src",
5037   .vector_size = sizeof (u32),
5038   .type = VLIB_NODE_TYPE_INTERNAL,
5039   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5040   .error_strings = snat_in2out_error_strings,
5041   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5042   .next_nodes = {
5043      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5044      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
5045      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5046      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5047   },
5048 };
5049
5050 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
5051                               snat_hairpin_src_fn);
5052
5053 static uword
5054 nat44_ed_hairpin_src_fn (vlib_main_t * vm,
5055                          vlib_node_runtime_t * node,
5056                          vlib_frame_t * frame)
5057 {
5058   return snat_hairpin_src_fn_inline (vm, node, frame, 1);
5059 }
5060
5061 VLIB_REGISTER_NODE (nat44_ed_hairpin_src_node) = {
5062   .function = nat44_ed_hairpin_src_fn,
5063   .name = "nat44-ed-hairpin-src",
5064   .vector_size = sizeof (u32),
5065   .type = VLIB_NODE_TYPE_INTERNAL,
5066   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5067   .error_strings = snat_in2out_error_strings,
5068   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
5069   .next_nodes = {
5070      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
5071      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-ed-in2out-output",
5072      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
5073      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
5074   },
5075 };
5076
5077 VLIB_NODE_FUNCTION_MULTIARCH (nat44_ed_hairpin_src_node,
5078                               nat44_ed_hairpin_src_fn);
5079
5080 static uword
5081 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
5082                                 vlib_node_runtime_t * node,
5083                                 vlib_frame_t * frame)
5084 {
5085   u32 n_left_from, * from, * to_next;
5086   snat_in2out_next_t next_index;
5087   u32 pkts_processed = 0;
5088   snat_main_t * sm = &snat_main;
5089   u32 stats_node_index;
5090
5091   stats_node_index = snat_in2out_fast_node.index;
5092
5093   from = vlib_frame_vector_args (frame);
5094   n_left_from = frame->n_vectors;
5095   next_index = node->cached_next_index;
5096
5097   while (n_left_from > 0)
5098     {
5099       u32 n_left_to_next;
5100
5101       vlib_get_next_frame (vm, node, next_index,
5102                            to_next, n_left_to_next);
5103
5104       while (n_left_from > 0 && n_left_to_next > 0)
5105         {
5106           u32 bi0;
5107           vlib_buffer_t * b0;
5108           u32 next0;
5109           u32 sw_if_index0;
5110           ip4_header_t * ip0;
5111           ip_csum_t sum0;
5112           u32 new_addr0, old_addr0;
5113           u16 old_port0, new_port0;
5114           udp_header_t * udp0;
5115           tcp_header_t * tcp0;
5116           icmp46_header_t * icmp0;
5117           snat_session_key_t key0, sm0;
5118           u32 proto0;
5119           u32 rx_fib_index0;
5120
5121           /* speculatively enqueue b0 to the current next frame */
5122           bi0 = from[0];
5123           to_next[0] = bi0;
5124           from += 1;
5125           to_next += 1;
5126           n_left_from -= 1;
5127           n_left_to_next -= 1;
5128
5129           b0 = vlib_get_buffer (vm, bi0);
5130           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
5131
5132           ip0 = vlib_buffer_get_current (b0);
5133           udp0 = ip4_next_header (ip0);
5134           tcp0 = (tcp_header_t *) udp0;
5135           icmp0 = (icmp46_header_t *) udp0;
5136
5137           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
5138           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
5139
5140           if (PREDICT_FALSE(ip0->ttl == 1))
5141             {
5142               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
5143               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
5144                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
5145                                            0);
5146               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
5147               goto trace0;
5148             }
5149
5150           proto0 = ip_proto_to_snat_proto (ip0->protocol);
5151
5152           if (PREDICT_FALSE (proto0 == ~0))
5153               goto trace0;
5154
5155           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
5156             {
5157               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
5158                                   rx_fib_index0, node, next0, ~0, 0, 0);
5159               goto trace0;
5160             }
5161
5162           key0.addr = ip0->src_address;
5163           key0.protocol = proto0;
5164           key0.port = udp0->src_port;
5165           key0.fib_index = rx_fib_index0;
5166
5167           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
5168             {
5169               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
5170               next0= SNAT_IN2OUT_NEXT_DROP;
5171               goto trace0;
5172             }
5173
5174           new_addr0 = sm0.addr.as_u32;
5175           new_port0 = sm0.port;
5176           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
5177           old_addr0 = ip0->src_address.as_u32;
5178           ip0->src_address.as_u32 = new_addr0;
5179
5180           sum0 = ip0->checksum;
5181           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5182                                  ip4_header_t,
5183                                  src_address /* changed member */);
5184           ip0->checksum = ip_csum_fold (sum0);
5185
5186           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
5187             {
5188               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5189                 {
5190                   old_port0 = tcp0->src_port;
5191                   tcp0->src_port = new_port0;
5192
5193                   sum0 = tcp0->checksum;
5194                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5195                                          ip4_header_t,
5196                                          dst_address /* changed member */);
5197                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
5198                                          ip4_header_t /* cheat */,
5199                                          length /* changed member */);
5200                   tcp0->checksum = ip_csum_fold(sum0);
5201                 }
5202               else
5203                 {
5204                   old_port0 = udp0->src_port;
5205                   udp0->src_port = new_port0;
5206                   udp0->checksum = 0;
5207                 }
5208             }
5209           else
5210             {
5211               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
5212                 {
5213                   sum0 = tcp0->checksum;
5214                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
5215                                          ip4_header_t,
5216                                          dst_address /* changed member */);
5217                   tcp0->checksum = ip_csum_fold(sum0);
5218                 }
5219             }
5220
5221           /* Hairpinning */
5222           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0, 0);
5223
5224         trace0:
5225           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
5226                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
5227             {
5228               snat_in2out_trace_t *t =
5229                  vlib_add_trace (vm, node, b0, sizeof (*t));
5230               t->sw_if_index = sw_if_index0;
5231               t->next_index = next0;
5232             }
5233
5234           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
5235
5236           /* verify speculative enqueue, maybe switch current next frame */
5237           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
5238                                            to_next, n_left_to_next,
5239                                            bi0, next0);
5240         }
5241
5242       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
5243     }
5244
5245   vlib_node_increment_counter (vm, stats_node_index,
5246                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
5247                                pkts_processed);
5248   return frame->n_vectors;
5249 }
5250
5251
5252 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
5253   .function = snat_in2out_fast_static_map_fn,
5254   .name = "nat44-in2out-fast",
5255   .vector_size = sizeof (u32),
5256   .format_trace = format_snat_in2out_fast_trace,
5257   .type = VLIB_NODE_TYPE_INTERNAL,
5258
5259   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
5260   .error_strings = snat_in2out_error_strings,
5261
5262   .runtime_data_bytes = sizeof (snat_runtime_t),
5263
5264   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
5265
5266   /* edit / add dispositions here */
5267   .next_nodes = {
5268     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
5269     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
5270     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
5271     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
5272     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
5273   },
5274 };
5275
5276 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);