src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37   u32 is_slow_path;
  38 } snat_in2out_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_in2out_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  51   char * tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  66
  67   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  68               t->sw_if_index, t->next_index);
  69
  70   return s;
  71 }
  72
  73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   snat_in2out_worker_handoff_trace_t * t =
  78     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  79   char * m;
  80
  81   m = t->do_handoff ? "next worker" : "same worker";
  82   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  83
  84   return s;
  85 }
  86
  87 typedef struct {
  88   u32 sw_if_index;
  89   u32 next_index;
  90   u8 cached;
  91 } nat44_in2out_reass_trace_t;
  92
  93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  94 {
  95   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  96   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  97   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  98
  99   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 100               t->sw_if_index, t->next_index,
 101               t->cached ? "cached" : "translated");
 102
 103   return s;
 104 }
 105
 106 vlib_node_registration_t snat_in2out_node;
 107 vlib_node_registration_t snat_in2out_slowpath_node;
 108 vlib_node_registration_t snat_in2out_fast_node;
 109 vlib_node_registration_t snat_in2out_worker_handoff_node;
 110 vlib_node_registration_t snat_det_in2out_node;
 111 vlib_node_registration_t snat_in2out_output_node;
 112 vlib_node_registration_t snat_in2out_output_slowpath_node;
 113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 114 vlib_node_registration_t snat_hairpin_dst_node;
 115 vlib_node_registration_t snat_hairpin_src_node;
 116 vlib_node_registration_t nat44_hairpinning_node;
 117 vlib_node_registration_t nat44_in2out_reass_node;
 118
 119
 120 #define foreach_snat_in2out_error                       \
 121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 122 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 123 _(OUT_OF_PORTS, "Out of ports")                         \
 124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 125 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 126 _(NO_TRANSLATION, "No translation")                     \
 127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 128 _(DROP_FRAGMENT, "Drop fragment")                       \
 129 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 131
 132 typedef enum {
 133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 134   foreach_snat_in2out_error
 135 #undef _
 136   SNAT_IN2OUT_N_ERROR,
 137 } snat_in2out_error_t;
 138
 139 static char * snat_in2out_error_strings[] = {
 140 #define _(sym,string) string,
 141   foreach_snat_in2out_error
 142 #undef _
 143 };
 144
 145 typedef enum {
 146   SNAT_IN2OUT_NEXT_LOOKUP,
 147   SNAT_IN2OUT_NEXT_DROP,
 148   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 149   SNAT_IN2OUT_NEXT_SLOW_PATH,
 150   SNAT_IN2OUT_NEXT_REASS,
 151   SNAT_IN2OUT_N_NEXT,
 152 } snat_in2out_next_t;
 153
 154 typedef enum {
 155   SNAT_HAIRPIN_SRC_NEXT_DROP,
 156   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 157   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 158   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 159   SNAT_HAIRPIN_SRC_N_NEXT,
 160 } snat_hairpin_next_t;
 161
 162 /**
 163  * @brief Check if packet should be translated
 164  *
 165  * Packets aimed at outside interface and external address with active session
 166  * should be translated.
 167  *
 168  * @param sm            NAT main
 169  * @param rt            NAT runtime data
 170  * @param sw_if_index0  index of the inside interface
 171  * @param ip0           IPv4 header
 172  * @param proto0        NAT protocol
 173  * @param rx_fib_index0 RX FIB index
 174  *
 175  * @returns 0 if packet should be translated otherwise 1
 176  */
 177 static inline int
 178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 179                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                          u32 rx_fib_index0)
 181 {
 182   if (sm->out2in_dpo)
 183     return 0;
 184
 185   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 186   fib_prefix_t pfx = {
 187     .fp_proto = FIB_PROTOCOL_IP4,
 188     .fp_len = 32,
 189     .fp_addr = {
 190         .ip4.as_u32 = ip0->dst_address.as_u32,
 191     },
 192   };
 193
 194   /* Don't NAT packet aimed at the intfc address */
 195   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 196                                       ip0->dst_address.as_u32)))
 197     return 1;
 198
 199   fei = fib_table_lookup (rx_fib_index0, &pfx);
 200   if (FIB_NODE_INDEX_INVALID != fei)
 201     {
 202       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 203       if (sw_if_index == ~0)
 204         {
 205           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 206           if (FIB_NODE_INDEX_INVALID != fei)
 207             sw_if_index = fib_entry_get_resolving_interface (fei);
 208         }
 209       snat_interface_t *i;
 210       pool_foreach (i, sm->interfaces,
 211       ({
 212         /* NAT packet aimed at outside interface */
 213         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 214           return 0;
 215       }));
 216     }
 217
 218   return 1;
 219 }
 220
 221 static inline int
 222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 223                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 224                     u32 rx_fib_index0, u32 thread_index)
 225 {
 226   udp_header_t * udp0 = ip4_next_header (ip0);
 227   snat_session_key_t key0, sm0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229
 230   key0.addr = ip0->dst_address;
 231   key0.port = udp0->dst_port;
 232   key0.protocol = proto0;
 233   key0.fib_index = sm->outside_fib_index;
 234   kv0.key = key0.as_u64;
 235
 236   /* NAT packet aimed at external address if */
 237   /* has active sessions */
 238   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 239                               &value0))
 240     {
 241       /* or is static mappings */
 242       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
 243         return 0;
 244     }
 245   else
 246     return 0;
 247
 248   if (sm->forwarding_enabled)
 249     return 1;
 250
 251   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 252                                  rx_fib_index0);
 253 }
 254
 255 static inline int
 256 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 257                                   u32 proto0, u16 src_port, u16 dst_port,
 258                                   u32 thread_index, u32 sw_if_index)
 259 {
 260   snat_session_key_t key0;
 261   clib_bihash_kv_8_8_t kv0, value0;
 262   snat_interface_t *i;
 263
 264   /* src NAT check */
 265   key0.addr = ip0->src_address;
 266   key0.port = src_port;
 267   key0.protocol = proto0;
 268   key0.fib_index = sm->outside_fib_index;
 269   kv0.key = key0.as_u64;
 270
 271   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 272                                &value0))
 273     return 1;
 274
 275   /* dst NAT check */
 276   key0.addr = ip0->dst_address;
 277   key0.port = dst_port;
 278   key0.protocol = proto0;
 279   key0.fib_index = sm->inside_fib_index;
 280   kv0.key = key0.as_u64;
 281   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 282                                &value0))
 283   {
 284     /* hairpinning */
 285     pool_foreach (i, sm->output_feature_interfaces,
 286     ({
 287       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 288         return 0;
 289     }));
 290     return 1;
 291   }
 292
 293   return 0;
 294 }
 295
 296 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 297                       ip4_header_t * ip0,
 298                       u32 rx_fib_index0,
 299                       snat_session_key_t * key0,
 300                       snat_session_t ** sessionp,
 301                       vlib_node_runtime_t * node,
 302                       u32 next0,
 303                       u32 thread_index)
 304 {
 305   snat_user_t *u;
 306   snat_session_t *s;
 307   clib_bihash_kv_8_8_t kv0;
 308   snat_session_key_t key1;
 309   u32 address_index = ~0;
 310   u32 outside_fib_index;
 311   uword * p;
 312   udp_header_t * udp0 = ip4_next_header (ip0);
 313   u8 is_sm = 0;
 314
 315   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 316     {
 317       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 318       nat_ipfix_logging_max_sessions(sm->max_translations);
 319       return SNAT_IN2OUT_NEXT_DROP;
 320     }
 321
 322   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 323   if (! p)
 324     {
 325       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 326       return SNAT_IN2OUT_NEXT_DROP;
 327     }
 328   outside_fib_index = p[0];
 329
 330   key1.protocol = key0->protocol;
 331
 332   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 333                               thread_index);
 334   if (!u)
 335     {
 336       clib_warning ("create NAT user failed");
 337       return SNAT_IN2OUT_NEXT_DROP;
 338     }
 339
 340   /* First try to match static mapping by local address and port */
 341   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0))
 342     {
 343       /* Try to create dynamic translation */
 344       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 345                                                thread_index, &key1,
 346                                                &address_index,
 347                                                sm->port_per_thread,
 348                                                sm->per_thread_data[thread_index].snat_thread_index))
 349         {
 350           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 351           return SNAT_IN2OUT_NEXT_DROP;
 352         }
 353     }
 354   else
 355     is_sm = 1;
 356
 357   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 358   if (!s)
 359     {
 360       clib_warning ("create NAT session failed");
 361       return SNAT_IN2OUT_NEXT_DROP;
 362     }
 363
 364   if (is_sm)
 365     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 366   user_session_increment (sm, u, is_sm);
 367   s->outside_address_index = address_index;
 368   s->in2out = *key0;
 369   s->out2in = key1;
 370   s->out2in.protocol = key0->protocol;
 371   s->out2in.fib_index = outside_fib_index;
 372   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 373   s->ext_host_port = udp0->dst_port;
 374   *sessionp = s;
 375
 376   /* Add to translation hashes */
 377   kv0.key = s->in2out.as_u64;
 378   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 379   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 380                                1 /* is_add */))
 381       clib_warning ("in2out key add failed");
 382
 383   kv0.key = s->out2in.as_u64;
 384   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 385
 386   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 387                                1 /* is_add */))
 388       clib_warning ("out2in key add failed");
 389
 390   /* log NAT event */
 391   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 392                                       s->out2in.addr.as_u32,
 393                                       s->in2out.protocol,
 394                                       s->in2out.port,
 395                                       s->out2in.port,
 396                                       s->in2out.fib_index);
 397   return next0;
 398 }
 399
 400 static_always_inline
 401 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 402                                  snat_session_key_t *p_key0)
 403 {
 404   icmp46_header_t *icmp0;
 405   snat_session_key_t key0;
 406   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 407   ip4_header_t *inner_ip0 = 0;
 408   void *l4_header = 0;
 409   icmp46_header_t *inner_icmp0;
 410
 411   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 412   echo0 = (icmp_echo_header_t *)(icmp0+1);
 413
 414   if (!icmp_is_error_message (icmp0))
 415     {
 416       key0.protocol = SNAT_PROTOCOL_ICMP;
 417       key0.addr = ip0->src_address;
 418       key0.port = echo0->identifier;
 419     }
 420   else
 421     {
 422       inner_ip0 = (ip4_header_t *)(echo0+1);
 423       l4_header = ip4_next_header (inner_ip0);
 424       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 425       key0.addr = inner_ip0->dst_address;
 426       switch (key0.protocol)
 427         {
 428         case SNAT_PROTOCOL_ICMP:
 429           inner_icmp0 = (icmp46_header_t*)l4_header;
 430           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 431           key0.port = inner_echo0->identifier;
 432           break;
 433         case SNAT_PROTOCOL_UDP:
 434         case SNAT_PROTOCOL_TCP:
 435           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 436           break;
 437         default:
 438           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 439         }
 440     }
 441   *p_key0 = key0;
 442   return -1; /* success */
 443 }
 444
 445 static_always_inline int
 446 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 447 {
 448   icmp46_header_t *icmp0;
 449   nat_ed_ses_key_t key0;
 450   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 451   ip4_header_t *inner_ip0 = 0;
 452   void *l4_header = 0;
 453   icmp46_header_t *inner_icmp0;
 454
 455   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 456   echo0 = (icmp_echo_header_t *)(icmp0+1);
 457
 458   if (!icmp_is_error_message (icmp0))
 459     {
 460       key0.proto = IP_PROTOCOL_ICMP;
 461       key0.l_addr = ip0->src_address;
 462       key0.r_addr = ip0->dst_address;
 463       key0.l_port = key0.r_port = echo0->identifier;
 464     }
 465   else
 466     {
 467       inner_ip0 = (ip4_header_t *)(echo0+1);
 468       l4_header = ip4_next_header (inner_ip0);
 469       key0.proto = inner_ip0->protocol;
 470       key0.r_addr = inner_ip0->src_address;
 471       key0.l_addr = inner_ip0->dst_address;
 472       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 473         {
 474         case SNAT_PROTOCOL_ICMP:
 475           inner_icmp0 = (icmp46_header_t*)l4_header;
 476           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 477           key0.r_port = key0.l_port = inner_echo0->identifier;
 478           break;
 479         case SNAT_PROTOCOL_UDP:
 480         case SNAT_PROTOCOL_TCP:
 481           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 482           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
 483           break;
 484         default:
 485           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 486         }
 487     }
 488   *p_key0 = key0;
 489   return 0;
 490 }
 491
 492 static inline int
 493 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip)
 494 {
 495   nat_ed_ses_key_t key;
 496   clib_bihash_kv_16_8_t kv, value;
 497   udp_header_t *udp;
 498
 499   if (!sm->forwarding_enabled)
 500     return 0;
 501
 502   if (ip->protocol == IP_PROTOCOL_ICMP)
 503     {
 504       if (icmp_get_ed_key (ip, &key))
 505         return 0;
 506     }
 507   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 508     {
 509       udp = ip4_next_header(ip);
 510       key.l_addr = ip->src_address;
 511       key.r_addr = ip->dst_address;
 512       key.proto = ip->protocol;
 513       key.r_port = udp->dst_port;
 514       key.l_port = udp->src_port;
 515     }
 516   else
 517     {
 518       key.l_addr = ip->src_address;
 519       key.r_addr = ip->dst_address;
 520       key.proto = ip->protocol;
 521       key.l_port = key.r_port = 0;
 522     }
 523   key.fib_index = 0;
 524   kv.key[0] = key.as_u64[0];
 525   kv.key[1] = key.as_u64[1];
 526
 527   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 528     return value.value == ~0ULL;
 529
 530   return 0;
 531 }
 532
 533 /**
 534  * Get address and port values to be used for ICMP packet translation
 535  * and create session if needed
 536  *
 537  * @param[in,out] sm             NAT main
 538  * @param[in,out] node           NAT node runtime
 539  * @param[in] thread_index       thread index
 540  * @param[in,out] b0             buffer containing packet to be translated
 541  * @param[out] p_proto           protocol used for matching
 542  * @param[out] p_value           address and port after NAT translation
 543  * @param[out] p_dont_translate  if packet should not be translated
 544  * @param d                      optional parameter
 545  * @param e                      optional parameter
 546  */
 547 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 548                            u32 thread_index, vlib_buffer_t *b0,
 549                            ip4_header_t *ip0, u8 *p_proto,
 550                            snat_session_key_t *p_value,
 551                            u8 *p_dont_translate, void *d, void *e)
 552 {
 553   icmp46_header_t *icmp0;
 554   u32 sw_if_index0;
 555   u32 rx_fib_index0;
 556   snat_session_key_t key0;
 557   snat_session_t *s0 = 0;
 558   u8 dont_translate = 0;
 559   clib_bihash_kv_8_8_t kv0, value0;
 560   u32 next0 = ~0;
 561   int err;
 562
 563   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 564   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 565   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 566
 567   err = icmp_get_key (ip0, &key0);
 568   if (err != -1)
 569     {
 570       b0->error = node->errors[err];
 571       next0 = SNAT_IN2OUT_NEXT_DROP;
 572       goto out;
 573     }
 574   key0.fib_index = rx_fib_index0;
 575
 576   kv0.key = key0.as_u64;
 577
 578   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 579                               &value0))
 580     {
 581       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 582         {
 583           if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
 584               ip0, SNAT_PROTOCOL_ICMP, key0.port, key0.port, thread_index, sw_if_index0)))
 585             {
 586               dont_translate = 1;
 587               goto out;
 588             }
 589         }
 590       else
 591         {
 592           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 593               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 594             {
 595               dont_translate = 1;
 596               goto out;
 597             }
 598         }
 599
 600       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 601         {
 602           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 603           next0 = SNAT_IN2OUT_NEXT_DROP;
 604           goto out;
 605         }
 606
 607       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 608                          &s0, node, next0, thread_index);
 609
 610       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 611         goto out;
 612     }
 613   else
 614     {
 615       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 616                         icmp0->type != ICMP4_echo_reply &&
 617                         !icmp_is_error_message (icmp0)))
 618         {
 619           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 620           next0 = SNAT_IN2OUT_NEXT_DROP;
 621           goto out;
 622         }
 623
 624       if (PREDICT_FALSE (value0.value == ~0ULL))
 625         {
 626           nat_ed_ses_key_t key;
 627           clib_bihash_kv_16_8_t s_kv, s_value;
 628
 629           key.as_u64[0] = 0;
 630           key.as_u64[1] = 0;
 631           if (icmp_get_ed_key (ip0, &key))
 632             {
 633               b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 634               next0 = SNAT_IN2OUT_NEXT_DROP;
 635               goto out;
 636             }
 637           key.fib_index = rx_fib_index0;
 638           s_kv.key[0] = key.as_u64[0];
 639           s_kv.key[1] = key.as_u64[1];
 640           if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 641             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 642                                     s_value.value);
 643           else
 644            {
 645               next0 = SNAT_IN2OUT_NEXT_DROP;
 646               goto out;
 647            }
 648         }
 649       else
 650         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 651                                 value0.value);
 652     }
 653
 654 out:
 655   *p_proto = key0.protocol;
 656   if (s0)
 657     *p_value = s0->out2in;
 658   *p_dont_translate = dont_translate;
 659   if (d)
 660     *(snat_session_t**)d = s0;
 661   return next0;
 662 }
 663
 664 /**
 665  * Get address and port values to be used for ICMP packet translation
 666  *
 667  * @param[in] sm                 NAT main
 668  * @param[in,out] node           NAT node runtime
 669  * @param[in] thread_index       thread index
 670  * @param[in,out] b0             buffer containing packet to be translated
 671  * @param[out] p_proto           protocol used for matching
 672  * @param[out] p_value           address and port after NAT translation
 673  * @param[out] p_dont_translate  if packet should not be translated
 674  * @param d                      optional parameter
 675  * @param e                      optional parameter
 676  */
 677 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 678                            u32 thread_index, vlib_buffer_t *b0,
 679                            ip4_header_t *ip0, u8 *p_proto,
 680                            snat_session_key_t *p_value,
 681                            u8 *p_dont_translate, void *d, void *e)
 682 {
 683   icmp46_header_t *icmp0;
 684   u32 sw_if_index0;
 685   u32 rx_fib_index0;
 686   snat_session_key_t key0;
 687   snat_session_key_t sm0;
 688   u8 dont_translate = 0;
 689   u8 is_addr_only;
 690   u32 next0 = ~0;
 691   int err;
 692
 693   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 694   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 695   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 696
 697   err = icmp_get_key (ip0, &key0);
 698   if (err != -1)
 699     {
 700       b0->error = node->errors[err];
 701       next0 = SNAT_IN2OUT_NEXT_DROP;
 702       goto out2;
 703     }
 704   key0.fib_index = rx_fib_index0;
 705
 706   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0))
 707     {
 708       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 709           IP_PROTOCOL_ICMP, rx_fib_index0)))
 710         {
 711           dont_translate = 1;
 712           goto out;
 713         }
 714
 715       if (icmp_is_error_message (icmp0))
 716         {
 717           next0 = SNAT_IN2OUT_NEXT_DROP;
 718           goto out;
 719         }
 720
 721       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 722       next0 = SNAT_IN2OUT_NEXT_DROP;
 723       goto out;
 724     }
 725
 726   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 727                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 728                     !icmp_is_error_message (icmp0)))
 729     {
 730       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 731       next0 = SNAT_IN2OUT_NEXT_DROP;
 732       goto out;
 733     }
 734
 735 out:
 736   *p_value = sm0;
 737 out2:
 738   *p_proto = key0.protocol;
 739   *p_dont_translate = dont_translate;
 740   return next0;
 741 }
 742
 743 static inline u32 icmp_in2out (snat_main_t *sm,
 744                                vlib_buffer_t * b0,
 745                                ip4_header_t * ip0,
 746                                icmp46_header_t * icmp0,
 747                                u32 sw_if_index0,
 748                                u32 rx_fib_index0,
 749                                vlib_node_runtime_t * node,
 750                                u32 next0,
 751                                u32 thread_index,
 752                                void *d,
 753                                void *e)
 754 {
 755   snat_session_key_t sm0;
 756   u8 protocol;
 757   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 758   ip4_header_t *inner_ip0;
 759   void *l4_header = 0;
 760   icmp46_header_t *inner_icmp0;
 761   u8 dont_translate;
 762   u32 new_addr0, old_addr0;
 763   u16 old_id0, new_id0;
 764   ip_csum_t sum0;
 765   u16 checksum0;
 766   u32 next0_tmp;
 767
 768   echo0 = (icmp_echo_header_t *)(icmp0+1);
 769
 770   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 771                                        &protocol, &sm0, &dont_translate, d, e);
 772   if (next0_tmp != ~0)
 773     next0 = next0_tmp;
 774   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 775     goto out;
 776
 777   sum0 = ip_incremental_checksum (0, icmp0,
 778                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 779   checksum0 = ~ip_csum_fold (sum0);
 780   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 781     {
 782       next0 = SNAT_IN2OUT_NEXT_DROP;
 783       goto out;
 784     }
 785
 786   old_addr0 = ip0->src_address.as_u32;
 787   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 788   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 789     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 790
 791   sum0 = ip0->checksum;
 792   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 793                          src_address /* changed member */);
 794   ip0->checksum = ip_csum_fold (sum0);
 795
 796   if (icmp0->checksum == 0)
 797     icmp0->checksum = 0xffff;
 798
 799   if (!icmp_is_error_message (icmp0))
 800     {
 801       new_id0 = sm0.port;
 802       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 803         {
 804           old_id0 = echo0->identifier;
 805           new_id0 = sm0.port;
 806           echo0->identifier = new_id0;
 807
 808           sum0 = icmp0->checksum;
 809           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 810                                  identifier);
 811           icmp0->checksum = ip_csum_fold (sum0);
 812         }
 813     }
 814   else
 815     {
 816       inner_ip0 = (ip4_header_t *)(echo0+1);
 817       l4_header = ip4_next_header (inner_ip0);
 818
 819       if (!ip4_header_checksum_is_valid (inner_ip0))
 820         {
 821           next0 = SNAT_IN2OUT_NEXT_DROP;
 822           goto out;
 823         }
 824
 825       old_addr0 = inner_ip0->dst_address.as_u32;
 826       inner_ip0->dst_address = sm0.addr;
 827       new_addr0 = inner_ip0->dst_address.as_u32;
 828
 829       sum0 = icmp0->checksum;
 830       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 831                              dst_address /* changed member */);
 832       icmp0->checksum = ip_csum_fold (sum0);
 833
 834       switch (protocol)
 835         {
 836           case SNAT_PROTOCOL_ICMP:
 837             inner_icmp0 = (icmp46_header_t*)l4_header;
 838             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 839
 840             old_id0 = inner_echo0->identifier;
 841             new_id0 = sm0.port;
 842             inner_echo0->identifier = new_id0;
 843
 844             sum0 = icmp0->checksum;
 845             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 846                                    identifier);
 847             icmp0->checksum = ip_csum_fold (sum0);
 848             break;
 849           case SNAT_PROTOCOL_UDP:
 850           case SNAT_PROTOCOL_TCP:
 851             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 852             new_id0 = sm0.port;
 853             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 854
 855             sum0 = icmp0->checksum;
 856             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 857                                    dst_port);
 858             icmp0->checksum = ip_csum_fold (sum0);
 859             break;
 860           default:
 861             ASSERT(0);
 862         }
 863     }
 864
 865 out:
 866   return next0;
 867 }
 868
 869 /**
 870  * @brief Hairpinning
 871  *
 872  * Hairpinning allows two endpoints on the internal side of the NAT to
 873  * communicate even if they only use each other's external IP addresses
 874  * and ports.
 875  *
 876  * @param sm     NAT main.
 877  * @param b0     Vlib buffer.
 878  * @param ip0    IP header.
 879  * @param udp0   UDP header.
 880  * @param tcp0   TCP header.
 881  * @param proto0 NAT protocol.
 882  */
 883 static inline int
 884 snat_hairpinning (snat_main_t *sm,
 885                   vlib_buffer_t * b0,
 886                   ip4_header_t * ip0,
 887                   udp_header_t * udp0,
 888                   tcp_header_t * tcp0,
 889                   u32 proto0)
 890 {
 891   snat_session_key_t key0, sm0;
 892   snat_session_t * s0;
 893   clib_bihash_kv_8_8_t kv0, value0;
 894   ip_csum_t sum0;
 895   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 896   u16 new_dst_port0, old_dst_port0;
 897
 898   key0.addr = ip0->dst_address;
 899   key0.port = udp0->dst_port;
 900   key0.protocol = proto0;
 901   key0.fib_index = sm->outside_fib_index;
 902   kv0.key = key0.as_u64;
 903
 904   /* Check if destination is static mappings */
 905   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
 906     {
 907       new_dst_addr0 = sm0.addr.as_u32;
 908       new_dst_port0 = sm0.port;
 909       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 910     }
 911   /* or active session */
 912   else
 913     {
 914       if (sm->num_workers > 1)
 915         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 916       else
 917         ti = sm->num_workers;
 918
 919       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 920         {
 921           si = value0.value;
 922
 923           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 924           new_dst_addr0 = s0->in2out.addr.as_u32;
 925           new_dst_port0 = s0->in2out.port;
 926           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 927         }
 928     }
 929
 930   /* Destination is behind the same NAT, use internal address and port */
 931   if (new_dst_addr0)
 932     {
 933       old_dst_addr0 = ip0->dst_address.as_u32;
 934       ip0->dst_address.as_u32 = new_dst_addr0;
 935       sum0 = ip0->checksum;
 936       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 937                              ip4_header_t, dst_address);
 938       ip0->checksum = ip_csum_fold (sum0);
 939
 940       old_dst_port0 = tcp0->dst;
 941       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 942         {
 943           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 944             {
 945               tcp0->dst = new_dst_port0;
 946               sum0 = tcp0->checksum;
 947               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 948                                      ip4_header_t, dst_address);
 949               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 950                                      ip4_header_t /* cheat */, length);
 951               tcp0->checksum = ip_csum_fold(sum0);
 952             }
 953           else
 954             {
 955               udp0->dst_port = new_dst_port0;
 956               udp0->checksum = 0;
 957             }
 958         }
 959       else
 960         {
 961           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 962             {
 963               sum0 = tcp0->checksum;
 964               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 965                                      ip4_header_t, dst_address);
 966               tcp0->checksum = ip_csum_fold(sum0);
 967             }
 968         }
 969       return 1;
 970     }
 971   return 0;
 972 }
 973
 974 static inline void
 975 snat_icmp_hairpinning (snat_main_t *sm,
 976                        vlib_buffer_t * b0,
 977                        ip4_header_t * ip0,
 978                        icmp46_header_t * icmp0)
 979 {
 980   snat_session_key_t key0, sm0;
 981   clib_bihash_kv_8_8_t kv0, value0;
 982   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 983   ip_csum_t sum0;
 984   snat_session_t *s0;
 985
 986   if (!icmp_is_error_message (icmp0))
 987     {
 988       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 989       u16 icmp_id0 = echo0->identifier;
 990       key0.addr = ip0->dst_address;
 991       key0.port = icmp_id0;
 992       key0.protocol = SNAT_PROTOCOL_ICMP;
 993       key0.fib_index = sm->outside_fib_index;
 994       kv0.key = key0.as_u64;
 995
 996       if (sm->num_workers > 1)
 997         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 998       else
 999         ti = sm->num_workers;
1000
1001       /* Check if destination is in active sessions */
1002       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1003                                   &value0))
1004         {
1005           /* or static mappings */
1006           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1007             {
1008               new_dst_addr0 = sm0.addr.as_u32;
1009               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1010             }
1011         }
1012       else
1013         {
1014           si = value0.value;
1015
1016           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1017           new_dst_addr0 = s0->in2out.addr.as_u32;
1018           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1019           echo0->identifier = s0->in2out.port;
1020           sum0 = icmp0->checksum;
1021           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1022                                  icmp_echo_header_t, identifier);
1023           icmp0->checksum = ip_csum_fold (sum0);
1024         }
1025
1026       /* Destination is behind the same NAT, use internal address and port */
1027       if (new_dst_addr0)
1028         {
1029           old_dst_addr0 = ip0->dst_address.as_u32;
1030           ip0->dst_address.as_u32 = new_dst_addr0;
1031           sum0 = ip0->checksum;
1032           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1033                                  ip4_header_t, dst_address);
1034           ip0->checksum = ip_csum_fold (sum0);
1035         }
1036     }
1037
1038 }
1039
1040 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1041                                          vlib_buffer_t * b0,
1042                                          ip4_header_t * ip0,
1043                                          icmp46_header_t * icmp0,
1044                                          u32 sw_if_index0,
1045                                          u32 rx_fib_index0,
1046                                          vlib_node_runtime_t * node,
1047                                          u32 next0,
1048                                          f64 now,
1049                                          u32 thread_index,
1050                                          snat_session_t ** p_s0)
1051 {
1052   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1053                       next0, thread_index, p_s0, 0);
1054   snat_session_t * s0 = *p_s0;
1055   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1056     {
1057       /* Hairpinning */
1058       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1059         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1060       /* Accounting */
1061       s0->last_heard = now;
1062       s0->total_pkts++;
1063       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1064       /* Per-user LRU list maintenance */
1065       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1066                          s0->per_user_index);
1067       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1068                           s0->per_user_list_head_index,
1069                           s0->per_user_index);
1070     }
1071   return next0;
1072 }
1073 static inline void
1074 snat_hairpinning_unknown_proto (snat_main_t *sm,
1075                                 vlib_buffer_t * b,
1076                                 ip4_header_t * ip)
1077 {
1078   u32 old_addr, new_addr = 0, ti = 0;
1079   clib_bihash_kv_8_8_t kv, value;
1080   clib_bihash_kv_16_8_t s_kv, s_value;
1081   nat_ed_ses_key_t key;
1082   snat_session_key_t m_key;
1083   snat_static_mapping_t *m;
1084   ip_csum_t sum;
1085   snat_session_t *s;
1086
1087   old_addr = ip->dst_address.as_u32;
1088   key.l_addr.as_u32 = ip->dst_address.as_u32;
1089   key.r_addr.as_u32 = ip->src_address.as_u32;
1090   key.fib_index = sm->outside_fib_index;
1091   key.proto = ip->protocol;
1092   key.r_port = 0;
1093   key.l_port = 0;
1094   s_kv.key[0] = key.as_u64[0];
1095   s_kv.key[1] = key.as_u64[1];
1096   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1097     {
1098       m_key.addr = ip->dst_address;
1099       m_key.fib_index = sm->outside_fib_index;
1100       m_key.port = 0;
1101       m_key.protocol = 0;
1102       kv.key = m_key.as_u64;
1103       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1104         return;
1105
1106       m = pool_elt_at_index (sm->static_mappings, value.value);
1107       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1108         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1109       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1110     }
1111   else
1112     {
1113       if (sm->num_workers > 1)
1114         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1115       else
1116         ti = sm->num_workers;
1117
1118       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1119       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1120         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1121       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1122     }
1123   sum = ip->checksum;
1124   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1125   ip->checksum = ip_csum_fold (sum);
1126 }
1127
1128 static snat_session_t *
1129 snat_in2out_unknown_proto (snat_main_t *sm,
1130                            vlib_buffer_t * b,
1131                            ip4_header_t * ip,
1132                            u32 rx_fib_index,
1133                            u32 thread_index,
1134                            f64 now,
1135                            vlib_main_t * vm,
1136                            vlib_node_runtime_t * node)
1137 {
1138   clib_bihash_kv_8_8_t kv, value;
1139   clib_bihash_kv_16_8_t s_kv, s_value;
1140   snat_static_mapping_t *m;
1141   snat_session_key_t m_key;
1142   u32 old_addr, new_addr = 0;
1143   ip_csum_t sum;
1144   snat_user_t *u;
1145   dlist_elt_t *head, *elt;
1146   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1147   u32 elt_index, head_index, ses_index;
1148   snat_session_t * s;
1149   nat_ed_ses_key_t key;
1150   u32 address_index = ~0;
1151   int i;
1152   u8 is_sm = 0;
1153
1154   old_addr = ip->src_address.as_u32;
1155
1156   key.l_addr = ip->src_address;
1157   key.r_addr = ip->dst_address;
1158   key.fib_index = rx_fib_index;
1159   key.proto = ip->protocol;
1160   key.l_port = 0;
1161   key.r_port = 0;
1162   s_kv.key[0] = key.as_u64[0];
1163   s_kv.key[1] = key.as_u64[1];
1164
1165   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1166     {
1167       s = pool_elt_at_index (tsm->sessions, s_value.value);
1168       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1169     }
1170   else
1171     {
1172       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1173         {
1174           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1175           nat_ipfix_logging_max_sessions(sm->max_translations);
1176           return 0;
1177         }
1178
1179       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1180                                   thread_index);
1181       if (!u)
1182         {
1183           clib_warning ("create NAT user failed");
1184           return 0;
1185         }
1186
1187       m_key.addr = ip->src_address;
1188       m_key.port = 0;
1189       m_key.protocol = 0;
1190       m_key.fib_index = rx_fib_index;
1191       kv.key = m_key.as_u64;
1192
1193       /* Try to find static mapping first */
1194       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1195         {
1196           m = pool_elt_at_index (sm->static_mappings, value.value);
1197           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1198           is_sm = 1;
1199           goto create_ses;
1200         }
1201       /* Fallback to 3-tuple key */
1202       else
1203         {
1204           /* Choose same out address as for TCP/UDP session to same destination */
1205           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1206             {
1207               head_index = u->sessions_per_user_list_head_index;
1208               head = pool_elt_at_index (tsm->list_pool, head_index);
1209               elt_index = head->next;
1210               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1211               ses_index = elt->value;
1212               while (ses_index != ~0)
1213                 {
1214                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1215                   elt_index = elt->next;
1216                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1217                   ses_index = elt->value;
1218
1219                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1220                     {
1221                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1222                       address_index = s->outside_address_index;
1223
1224                       key.fib_index = sm->outside_fib_index;
1225                       key.l_addr.as_u32 = new_addr;
1226                       s_kv.key[0] = key.as_u64[0];
1227                       s_kv.key[1] = key.as_u64[1];
1228                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1229                         break;
1230
1231                       goto create_ses;
1232                     }
1233                 }
1234             }
1235           key.fib_index = sm->outside_fib_index;
1236           for (i = 0; i < vec_len (sm->addresses); i++)
1237             {
1238               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1239               s_kv.key[0] = key.as_u64[0];
1240               s_kv.key[1] = key.as_u64[1];
1241               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1242                 {
1243                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1244                   address_index = i;
1245                   goto create_ses;
1246                 }
1247             }
1248           return 0;
1249         }
1250
1251 create_ses:
1252       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1253       if (!s)
1254         {
1255           clib_warning ("create NAT session failed");
1256           return 0;
1257         }
1258
1259       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1260       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1261       s->outside_address_index = address_index;
1262       s->out2in.addr.as_u32 = new_addr;
1263       s->out2in.fib_index = sm->outside_fib_index;
1264       s->in2out.addr.as_u32 = old_addr;
1265       s->in2out.fib_index = rx_fib_index;
1266       s->in2out.port = s->out2in.port = ip->protocol;
1267       if (is_sm)
1268         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1269       user_session_increment (sm, u, is_sm);
1270
1271       /* Add to lookup tables */
1272       key.l_addr.as_u32 = old_addr;
1273       key.r_addr = ip->dst_address;
1274       key.proto = ip->protocol;
1275       key.fib_index = rx_fib_index;
1276       s_kv.key[0] = key.as_u64[0];
1277       s_kv.key[1] = key.as_u64[1];
1278       s_kv.value = s - tsm->sessions;
1279       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1280         clib_warning ("in2out key add failed");
1281
1282       key.l_addr.as_u32 = new_addr;
1283       key.fib_index = sm->outside_fib_index;
1284       s_kv.key[0] = key.as_u64[0];
1285       s_kv.key[1] = key.as_u64[1];
1286       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1287         clib_warning ("out2in key add failed");
1288   }
1289
1290   /* Update IP checksum */
1291   sum = ip->checksum;
1292   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1293   ip->checksum = ip_csum_fold (sum);
1294
1295   /* Accounting */
1296   s->last_heard = now;
1297   s->total_pkts++;
1298   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1299   /* Per-user LRU list maintenance */
1300   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1301   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1302                       s->per_user_index);
1303
1304   /* Hairpinning */
1305   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1306     snat_hairpinning_unknown_proto(sm, b, ip);
1307
1308   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1309     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1310
1311   return s;
1312 }
1313
1314 static snat_session_t *
1315 snat_in2out_lb (snat_main_t *sm,
1316                 vlib_buffer_t * b,
1317                 ip4_header_t * ip,
1318                 u32 rx_fib_index,
1319                 u32 thread_index,
1320                 f64 now,
1321                 vlib_main_t * vm,
1322                 vlib_node_runtime_t * node)
1323 {
1324   nat_ed_ses_key_t key;
1325   clib_bihash_kv_16_8_t s_kv, s_value;
1326   udp_header_t *udp = ip4_next_header (ip);
1327   tcp_header_t *tcp = (tcp_header_t *) udp;
1328   snat_session_t *s = 0;
1329   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1330   u32 old_addr, new_addr;
1331   u16 new_port, old_port;
1332   ip_csum_t sum;
1333   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1334   snat_session_key_t e_key, l_key;
1335   snat_user_t *u;
1336
1337   old_addr = ip->src_address.as_u32;
1338
1339   key.l_addr = ip->src_address;
1340   key.r_addr = ip->dst_address;
1341   key.fib_index = rx_fib_index;
1342   key.proto = ip->protocol;
1343   key.r_port = udp->dst_port;
1344   key.l_port = udp->src_port;
1345   s_kv.key[0] = key.as_u64[0];
1346   s_kv.key[1] = key.as_u64[1];
1347
1348   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1349     {
1350       if (s_value.value == ~0ULL)
1351         return 0;
1352       s = pool_elt_at_index (tsm->sessions, s_value.value);
1353     }
1354   else
1355     {
1356       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1357         {
1358           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1359           nat_ipfix_logging_max_sessions(sm->max_translations);
1360           return 0;
1361         }
1362
1363       l_key.addr = ip->src_address;
1364       l_key.port = udp->src_port;
1365       l_key.protocol = proto;
1366       l_key.fib_index = rx_fib_index;
1367       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0))
1368         return 0;
1369
1370       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1371                                   thread_index);
1372       if (!u)
1373         {
1374           clib_warning ("create NAT user failed");
1375           return 0;
1376         }
1377
1378       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1379       if (!s)
1380         {
1381           clib_warning ("create NAT session failed");
1382           return 0;
1383         }
1384
1385       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1386       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1387       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1388       s->outside_address_index = ~0;
1389       s->in2out = l_key;
1390       s->out2in = e_key;
1391       s->out2in.protocol = l_key.protocol;
1392       user_session_increment (sm, u, 1 /* static */);
1393
1394       /* Add to lookup tables */
1395       s_kv.value = s - tsm->sessions;
1396       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1397         clib_warning ("in2out-ed key add failed");
1398
1399       key.l_addr = e_key.addr;
1400       key.fib_index = e_key.fib_index;
1401       key.l_port = e_key.port;
1402       s_kv.key[0] = key.as_u64[0];
1403       s_kv.key[1] = key.as_u64[1];
1404       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1405         clib_warning ("out2in-ed key add failed");
1406     }
1407
1408   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1409
1410   /* Update IP checksum */
1411   sum = ip->checksum;
1412   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1413   if (is_twice_nat_session (s))
1414     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1415                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1416   ip->checksum = ip_csum_fold (sum);
1417
1418   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1419     {
1420       old_port = tcp->src_port;
1421       tcp->src_port = s->out2in.port;
1422       new_port = tcp->src_port;
1423
1424       sum = tcp->checksum;
1425       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1426       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1427       if (is_twice_nat_session (s))
1428         {
1429           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1430                                 s->ext_host_addr.as_u32, ip4_header_t,
1431                                 dst_address);
1432           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1433                                 ip4_header_t, length);
1434           tcp->dst_port = s->ext_host_port;
1435           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1436         }
1437       tcp->checksum = ip_csum_fold(sum);
1438     }
1439   else
1440     {
1441       udp->src_port = s->out2in.port;
1442       if (is_twice_nat_session (s))
1443         {
1444           udp->dst_port = s->ext_host_port;
1445           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1446         }
1447       udp->checksum = 0;
1448     }
1449
1450   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1451     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1452
1453   /* Accounting */
1454   s->last_heard = now;
1455   s->total_pkts++;
1456   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1457   /* Per-user LRU list maintenance */
1458   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1459   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1460                       s->per_user_index);
1461   return s;
1462 }
1463
1464 static inline uword
1465 snat_in2out_node_fn_inline (vlib_main_t * vm,
1466                             vlib_node_runtime_t * node,
1467                             vlib_frame_t * frame, int is_slow_path,
1468                             int is_output_feature)
1469 {
1470   u32 n_left_from, * from, * to_next;
1471   snat_in2out_next_t next_index;
1472   u32 pkts_processed = 0;
1473   snat_main_t * sm = &snat_main;
1474   f64 now = vlib_time_now (vm);
1475   u32 stats_node_index;
1476   u32 thread_index = vlib_get_thread_index ();
1477
1478   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1479     snat_in2out_node.index;
1480
1481   from = vlib_frame_vector_args (frame);
1482   n_left_from = frame->n_vectors;
1483   next_index = node->cached_next_index;
1484
1485   while (n_left_from > 0)
1486     {
1487       u32 n_left_to_next;
1488
1489       vlib_get_next_frame (vm, node, next_index,
1490                            to_next, n_left_to_next);
1491
1492       while (n_left_from >= 4 && n_left_to_next >= 2)
1493         {
1494           u32 bi0, bi1;
1495           vlib_buffer_t * b0, * b1;
1496           u32 next0, next1;
1497           u32 sw_if_index0, sw_if_index1;
1498           ip4_header_t * ip0, * ip1;
1499           ip_csum_t sum0, sum1;
1500           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1501           u16 old_port0, new_port0, old_port1, new_port1;
1502           udp_header_t * udp0, * udp1;
1503           tcp_header_t * tcp0, * tcp1;
1504           icmp46_header_t * icmp0, * icmp1;
1505           snat_session_key_t key0, key1;
1506           u32 rx_fib_index0, rx_fib_index1;
1507           u32 proto0, proto1;
1508           snat_session_t * s0 = 0, * s1 = 0;
1509           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1510           u32 iph_offset0 = 0, iph_offset1 = 0;
1511
1512           /* Prefetch next iteration. */
1513           {
1514             vlib_buffer_t * p2, * p3;
1515
1516             p2 = vlib_get_buffer (vm, from[2]);
1517             p3 = vlib_get_buffer (vm, from[3]);
1518
1519             vlib_prefetch_buffer_header (p2, LOAD);
1520             vlib_prefetch_buffer_header (p3, LOAD);
1521
1522             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1523             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1524           }
1525
1526           /* speculatively enqueue b0 and b1 to the current next frame */
1527           to_next[0] = bi0 = from[0];
1528           to_next[1] = bi1 = from[1];
1529           from += 2;
1530           to_next += 2;
1531           n_left_from -= 2;
1532           n_left_to_next -= 2;
1533
1534           b0 = vlib_get_buffer (vm, bi0);
1535           b1 = vlib_get_buffer (vm, bi1);
1536
1537           if (is_output_feature)
1538             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1539
1540           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1541                  iph_offset0);
1542
1543           udp0 = ip4_next_header (ip0);
1544           tcp0 = (tcp_header_t *) udp0;
1545           icmp0 = (icmp46_header_t *) udp0;
1546
1547           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1548           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1549                                    sw_if_index0);
1550
1551           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1552
1553           if (PREDICT_FALSE(ip0->ttl == 1))
1554             {
1555               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1556               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1557                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1558                                            0);
1559               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1560               goto trace00;
1561             }
1562
1563           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1564
1565           /* Next configured feature, probably ip4-lookup */
1566           if (is_slow_path)
1567             {
1568               if (PREDICT_FALSE (proto0 == ~0))
1569                 {
1570                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1571                                                   thread_index, now, vm, node);
1572                   if (!s0)
1573                     next0 = SNAT_IN2OUT_NEXT_DROP;
1574                   goto trace00;
1575                 }
1576
1577               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1578                 {
1579                   next0 = icmp_in2out_slow_path
1580                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1581                      node, next0, now, thread_index, &s0);
1582                   goto trace00;
1583                 }
1584             }
1585           else
1586             {
1587               if (is_output_feature)
1588                 {
1589                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0)))
1590                     goto trace00;
1591                 }
1592
1593               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1594                 {
1595                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1596                   goto trace00;
1597                 }
1598
1599               if (ip4_is_fragment (ip0))
1600                 {
1601                   next0 = SNAT_IN2OUT_NEXT_REASS;
1602                   goto trace00;
1603                 }
1604             }
1605
1606           key0.addr = ip0->src_address;
1607           key0.port = udp0->src_port;
1608           key0.protocol = proto0;
1609           key0.fib_index = rx_fib_index0;
1610
1611           kv0.key = key0.as_u64;
1612
1613           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1614               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1615             {
1616               if (is_slow_path)
1617                 {
1618                   if (is_output_feature)
1619                     {
1620                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1621                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1622                         goto trace00;
1623                     }
1624                   else
1625                     {
1626                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1627                           ip0, proto0, rx_fib_index0, thread_index)))
1628                         goto trace00;
1629                     }
1630
1631                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1632                                      &s0, node, next0, thread_index);
1633                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1634                     goto trace00;
1635                 }
1636               else
1637                 {
1638                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1639                   goto trace00;
1640                 }
1641             }
1642           else
1643             {
1644               if (PREDICT_FALSE (value0.value == ~0ULL))
1645                 {
1646                   if (is_slow_path)
1647                     {
1648                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1649                                           thread_index, now, vm, node);
1650                       if (!s0 && !sm->forwarding_enabled)
1651                         next0 = SNAT_IN2OUT_NEXT_DROP;
1652                       goto trace00;
1653                     }
1654                   else
1655                     {
1656                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1657                       goto trace00;
1658                     }
1659                 }
1660               else
1661                 {
1662                   s0 = pool_elt_at_index (
1663                     sm->per_thread_data[thread_index].sessions,
1664                     value0.value);
1665                 }
1666             }
1667
1668           b0->flags |= VNET_BUFFER_F_IS_NATED;
1669
1670           old_addr0 = ip0->src_address.as_u32;
1671           ip0->src_address = s0->out2in.addr;
1672           new_addr0 = ip0->src_address.as_u32;
1673           if (!is_output_feature)
1674             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1675
1676           sum0 = ip0->checksum;
1677           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1678                                  ip4_header_t,
1679                                  src_address /* changed member */);
1680           ip0->checksum = ip_csum_fold (sum0);
1681
1682           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1683             {
1684               old_port0 = tcp0->src_port;
1685               tcp0->src_port = s0->out2in.port;
1686               new_port0 = tcp0->src_port;
1687
1688               sum0 = tcp0->checksum;
1689               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1690                                      ip4_header_t,
1691                                      dst_address /* changed member */);
1692               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1693                                      ip4_header_t /* cheat */,
1694                                      length /* changed member */);
1695               tcp0->checksum = ip_csum_fold(sum0);
1696             }
1697           else
1698             {
1699               old_port0 = udp0->src_port;
1700               udp0->src_port = s0->out2in.port;
1701               udp0->checksum = 0;
1702             }
1703
1704           /* Accounting */
1705           s0->last_heard = now;
1706           s0->total_pkts++;
1707           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1708           /* Per-user LRU list maintenance */
1709           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1710                              s0->per_user_index);
1711           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1712                               s0->per_user_list_head_index,
1713                               s0->per_user_index);
1714         trace00:
1715
1716           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1717                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1718             {
1719               snat_in2out_trace_t *t =
1720                  vlib_add_trace (vm, node, b0, sizeof (*t));
1721               t->is_slow_path = is_slow_path;
1722               t->sw_if_index = sw_if_index0;
1723               t->next_index = next0;
1724                   t->session_index = ~0;
1725               if (s0)
1726                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1727             }
1728
1729           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1730
1731           if (is_output_feature)
1732             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1733
1734           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1735                  iph_offset1);
1736
1737           udp1 = ip4_next_header (ip1);
1738           tcp1 = (tcp_header_t *) udp1;
1739           icmp1 = (icmp46_header_t *) udp1;
1740
1741           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1742           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1743                                    sw_if_index1);
1744
1745           if (PREDICT_FALSE(ip1->ttl == 1))
1746             {
1747               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1748               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1749                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1750                                            0);
1751               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1752               goto trace01;
1753             }
1754
1755           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1756
1757           /* Next configured feature, probably ip4-lookup */
1758           if (is_slow_path)
1759             {
1760               if (PREDICT_FALSE (proto1 == ~0))
1761                 {
1762                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1763                                                   thread_index, now, vm, node);
1764                   if (!s1)
1765                     next1 = SNAT_IN2OUT_NEXT_DROP;
1766                   goto trace01;
1767                 }
1768
1769               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1770                 {
1771                   next1 = icmp_in2out_slow_path
1772                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1773                      next1, now, thread_index, &s1);
1774                   goto trace01;
1775                 }
1776             }
1777           else
1778             {
1779               if (is_output_feature)
1780                 {
1781                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip1)))
1782                     goto trace01;
1783                 }
1784
1785               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1786                 {
1787                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1788                   goto trace01;
1789                 }
1790
1791               if (ip4_is_fragment (ip1))
1792                 {
1793                   next1 = SNAT_IN2OUT_NEXT_REASS;
1794                   goto trace01;
1795                 }
1796             }
1797
1798           key1.addr = ip1->src_address;
1799           key1.port = udp1->src_port;
1800           key1.protocol = proto1;
1801           key1.fib_index = rx_fib_index1;
1802
1803           kv1.key = key1.as_u64;
1804
1805             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1806                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1807             {
1808               if (is_slow_path)
1809                 {
1810                   if (is_output_feature)
1811                     {
1812                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1813                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1814                         goto trace01;
1815                     }
1816                   else
1817                     {
1818                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1819                           ip1, proto1, rx_fib_index1, thread_index)))
1820                         goto trace01;
1821                     }
1822
1823                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1824                                      &s1, node, next1, thread_index);
1825                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1826                     goto trace01;
1827                 }
1828               else
1829                 {
1830                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1831                   goto trace01;
1832                 }
1833             }
1834           else
1835             {
1836               if (PREDICT_FALSE (value1.value == ~0ULL))
1837                 {
1838                   if (is_slow_path)
1839                     {
1840                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1841                                           thread_index, now, vm, node);
1842                       if (!s1 && !sm->forwarding_enabled)
1843                         next1 = SNAT_IN2OUT_NEXT_DROP;
1844                       goto trace01;
1845                     }
1846                   else
1847                     {
1848                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1849                       goto trace01;
1850                     }
1851                 }
1852               else
1853                 {
1854                   s1 = pool_elt_at_index (
1855                     sm->per_thread_data[thread_index].sessions,
1856                     value1.value);
1857                 }
1858             }
1859
1860           b1->flags |= VNET_BUFFER_F_IS_NATED;
1861
1862           old_addr1 = ip1->src_address.as_u32;
1863           ip1->src_address = s1->out2in.addr;
1864           new_addr1 = ip1->src_address.as_u32;
1865           if (!is_output_feature)
1866             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1867
1868           sum1 = ip1->checksum;
1869           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1870                                  ip4_header_t,
1871                                  src_address /* changed member */);
1872           ip1->checksum = ip_csum_fold (sum1);
1873
1874           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1875             {
1876               old_port1 = tcp1->src_port;
1877               tcp1->src_port = s1->out2in.port;
1878               new_port1 = tcp1->src_port;
1879
1880               sum1 = tcp1->checksum;
1881               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1882                                      ip4_header_t,
1883                                      dst_address /* changed member */);
1884               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1885                                      ip4_header_t /* cheat */,
1886                                      length /* changed member */);
1887               tcp1->checksum = ip_csum_fold(sum1);
1888             }
1889           else
1890             {
1891               old_port1 = udp1->src_port;
1892               udp1->src_port = s1->out2in.port;
1893               udp1->checksum = 0;
1894             }
1895
1896           /* Accounting */
1897           s1->last_heard = now;
1898           s1->total_pkts++;
1899           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1900           /* Per-user LRU list maintenance */
1901           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1902                              s1->per_user_index);
1903           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1904                               s1->per_user_list_head_index,
1905                               s1->per_user_index);
1906         trace01:
1907
1908           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1909                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1910             {
1911               snat_in2out_trace_t *t =
1912                  vlib_add_trace (vm, node, b1, sizeof (*t));
1913               t->sw_if_index = sw_if_index1;
1914               t->next_index = next1;
1915               t->session_index = ~0;
1916               if (s1)
1917                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1918             }
1919
1920           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1921
1922           /* verify speculative enqueues, maybe switch current next frame */
1923           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1924                                            to_next, n_left_to_next,
1925                                            bi0, bi1, next0, next1);
1926         }
1927
1928       while (n_left_from > 0 && n_left_to_next > 0)
1929         {
1930           u32 bi0;
1931           vlib_buffer_t * b0;
1932           u32 next0;
1933           u32 sw_if_index0;
1934           ip4_header_t * ip0;
1935           ip_csum_t sum0;
1936           u32 new_addr0, old_addr0;
1937           u16 old_port0, new_port0;
1938           udp_header_t * udp0;
1939           tcp_header_t * tcp0;
1940           icmp46_header_t * icmp0;
1941           snat_session_key_t key0;
1942           u32 rx_fib_index0;
1943           u32 proto0;
1944           snat_session_t * s0 = 0;
1945           clib_bihash_kv_8_8_t kv0, value0;
1946           u32 iph_offset0 = 0;
1947
1948           /* speculatively enqueue b0 to the current next frame */
1949           bi0 = from[0];
1950           to_next[0] = bi0;
1951           from += 1;
1952           to_next += 1;
1953           n_left_from -= 1;
1954           n_left_to_next -= 1;
1955
1956           b0 = vlib_get_buffer (vm, bi0);
1957           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1958
1959           if (is_output_feature)
1960             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1961
1962           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1963                  iph_offset0);
1964
1965           udp0 = ip4_next_header (ip0);
1966           tcp0 = (tcp_header_t *) udp0;
1967           icmp0 = (icmp46_header_t *) udp0;
1968
1969           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1970           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1971                                    sw_if_index0);
1972
1973           if (PREDICT_FALSE(ip0->ttl == 1))
1974             {
1975               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1976               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1977                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1978                                            0);
1979               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1980               goto trace0;
1981             }
1982
1983           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1984
1985           /* Next configured feature, probably ip4-lookup */
1986           if (is_slow_path)
1987             {
1988               if (PREDICT_FALSE (proto0 == ~0))
1989                 {
1990                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1991                                                   thread_index, now, vm, node);
1992                   if (!s0)
1993                     next0 = SNAT_IN2OUT_NEXT_DROP;
1994                   goto trace0;
1995                 }
1996
1997               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1998                 {
1999                   next0 = icmp_in2out_slow_path
2000                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2001                      next0, now, thread_index, &s0);
2002                   goto trace0;
2003                 }
2004             }
2005           else
2006             {
2007                if (is_output_feature)
2008                 {
2009                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0)))
2010                     goto trace0;
2011                 }
2012
2013               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2014                 {
2015                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2016                   goto trace0;
2017                 }
2018
2019               if (ip4_is_fragment (ip0))
2020                 {
2021                   next0 = SNAT_IN2OUT_NEXT_REASS;
2022                   goto trace0;
2023                 }
2024             }
2025
2026           key0.addr = ip0->src_address;
2027           key0.port = udp0->src_port;
2028           key0.protocol = proto0;
2029           key0.fib_index = rx_fib_index0;
2030
2031           kv0.key = key0.as_u64;
2032
2033           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2034                                       &kv0, &value0))
2035             {
2036               if (is_slow_path)
2037                 {
2038                   if (is_output_feature)
2039                     {
2040                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
2041                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
2042                         goto trace0;
2043                     }
2044                   else
2045                     {
2046                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2047                           ip0, proto0, rx_fib_index0, thread_index)))
2048                         goto trace0;
2049                     }
2050
2051                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2052                                      &s0, node, next0, thread_index);
2053
2054                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2055                     goto trace0;
2056                 }
2057               else
2058                 {
2059                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2060                   goto trace0;
2061                 }
2062             }
2063           else
2064             {
2065               if (PREDICT_FALSE (value0.value == ~0ULL))
2066                 {
2067                   if (is_slow_path)
2068                     {
2069                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2070                                           thread_index, now, vm, node);
2071                       if (!s0 && !sm->forwarding_enabled)
2072                         next0 = SNAT_IN2OUT_NEXT_DROP;
2073                       goto trace0;
2074                     }
2075                   else
2076                     {
2077                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2078                       goto trace0;
2079                     }
2080                 }
2081               else
2082                 {
2083                   s0 = pool_elt_at_index (
2084                     sm->per_thread_data[thread_index].sessions,
2085                     value0.value);
2086                 }
2087             }
2088
2089           b0->flags |= VNET_BUFFER_F_IS_NATED;
2090
2091           old_addr0 = ip0->src_address.as_u32;
2092           ip0->src_address = s0->out2in.addr;
2093           new_addr0 = ip0->src_address.as_u32;
2094           if (!is_output_feature)
2095             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2096
2097           sum0 = ip0->checksum;
2098           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2099                                  ip4_header_t,
2100                                  src_address /* changed member */);
2101           ip0->checksum = ip_csum_fold (sum0);
2102
2103           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2104             {
2105               old_port0 = tcp0->src_port;
2106               tcp0->src_port = s0->out2in.port;
2107               new_port0 = tcp0->src_port;
2108
2109               sum0 = tcp0->checksum;
2110               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2111                                      ip4_header_t,
2112                                      dst_address /* changed member */);
2113               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2114                                      ip4_header_t /* cheat */,
2115                                      length /* changed member */);
2116               tcp0->checksum = ip_csum_fold(sum0);
2117             }
2118           else
2119             {
2120               old_port0 = udp0->src_port;
2121               udp0->src_port = s0->out2in.port;
2122               udp0->checksum = 0;
2123             }
2124
2125           /* Accounting */
2126           s0->last_heard = now;
2127           s0->total_pkts++;
2128           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2129           /* Per-user LRU list maintenance */
2130           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2131                              s0->per_user_index);
2132           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2133                               s0->per_user_list_head_index,
2134                               s0->per_user_index);
2135
2136         trace0:
2137           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2138                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2139             {
2140               snat_in2out_trace_t *t =
2141                  vlib_add_trace (vm, node, b0, sizeof (*t));
2142               t->is_slow_path = is_slow_path;
2143               t->sw_if_index = sw_if_index0;
2144               t->next_index = next0;
2145                   t->session_index = ~0;
2146               if (s0)
2147                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2148             }
2149
2150           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2151
2152           /* verify speculative enqueue, maybe switch current next frame */
2153           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2154                                            to_next, n_left_to_next,
2155                                            bi0, next0);
2156         }
2157
2158       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2159     }
2160
2161   vlib_node_increment_counter (vm, stats_node_index,
2162                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2163                                pkts_processed);
2164   return frame->n_vectors;
2165 }
2166
2167 static uword
2168 snat_in2out_fast_path_fn (vlib_main_t * vm,
2169                           vlib_node_runtime_t * node,
2170                           vlib_frame_t * frame)
2171 {
2172   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2173 }
2174
2175 VLIB_REGISTER_NODE (snat_in2out_node) = {
2176   .function = snat_in2out_fast_path_fn,
2177   .name = "nat44-in2out",
2178   .vector_size = sizeof (u32),
2179   .format_trace = format_snat_in2out_trace,
2180   .type = VLIB_NODE_TYPE_INTERNAL,
2181
2182   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2183   .error_strings = snat_in2out_error_strings,
2184
2185   .runtime_data_bytes = sizeof (snat_runtime_t),
2186
2187   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2188
2189   /* edit / add dispositions here */
2190   .next_nodes = {
2191     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2192     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2193     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2194     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2195     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2196   },
2197 };
2198
2199 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2200
2201 static uword
2202 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2203                                  vlib_node_runtime_t * node,
2204                                  vlib_frame_t * frame)
2205 {
2206   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2207 }
2208
2209 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2210   .function = snat_in2out_output_fast_path_fn,
2211   .name = "nat44-in2out-output",
2212   .vector_size = sizeof (u32),
2213   .format_trace = format_snat_in2out_trace,
2214   .type = VLIB_NODE_TYPE_INTERNAL,
2215
2216   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2217   .error_strings = snat_in2out_error_strings,
2218
2219   .runtime_data_bytes = sizeof (snat_runtime_t),
2220
2221   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2222
2223   /* edit / add dispositions here */
2224   .next_nodes = {
2225     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2226     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2227     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2228     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2229     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2230   },
2231 };
2232
2233 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2234                               snat_in2out_output_fast_path_fn);
2235
2236 static uword
2237 snat_in2out_slow_path_fn (vlib_main_t * vm,
2238                           vlib_node_runtime_t * node,
2239                           vlib_frame_t * frame)
2240 {
2241   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2242 }
2243
2244 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2245   .function = snat_in2out_slow_path_fn,
2246   .name = "nat44-in2out-slowpath",
2247   .vector_size = sizeof (u32),
2248   .format_trace = format_snat_in2out_trace,
2249   .type = VLIB_NODE_TYPE_INTERNAL,
2250
2251   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2252   .error_strings = snat_in2out_error_strings,
2253
2254   .runtime_data_bytes = sizeof (snat_runtime_t),
2255
2256   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2257
2258   /* edit / add dispositions here */
2259   .next_nodes = {
2260     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2261     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2262     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2263     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2264     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2265   },
2266 };
2267
2268 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2269                               snat_in2out_slow_path_fn);
2270
2271 static uword
2272 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2273                                  vlib_node_runtime_t * node,
2274                                  vlib_frame_t * frame)
2275 {
2276   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2277 }
2278
2279 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2280   .function = snat_in2out_output_slow_path_fn,
2281   .name = "nat44-in2out-output-slowpath",
2282   .vector_size = sizeof (u32),
2283   .format_trace = format_snat_in2out_trace,
2284   .type = VLIB_NODE_TYPE_INTERNAL,
2285
2286   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2287   .error_strings = snat_in2out_error_strings,
2288
2289   .runtime_data_bytes = sizeof (snat_runtime_t),
2290
2291   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2292
2293   /* edit / add dispositions here */
2294   .next_nodes = {
2295     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2296     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2297     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2298     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2299     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2300   },
2301 };
2302
2303 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2304                               snat_in2out_output_slow_path_fn);
2305
2306 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2307
2308 static uword
2309 nat44_hairpinning_fn (vlib_main_t * vm,
2310                       vlib_node_runtime_t * node,
2311                       vlib_frame_t * frame)
2312 {
2313   u32 n_left_from, * from, * to_next;
2314   snat_in2out_next_t next_index;
2315   u32 pkts_processed = 0;
2316   snat_main_t * sm = &snat_main;
2317   vnet_feature_main_t *fm = &feature_main;
2318   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2319   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2320
2321   from = vlib_frame_vector_args (frame);
2322   n_left_from = frame->n_vectors;
2323   next_index = node->cached_next_index;
2324
2325   while (n_left_from > 0)
2326     {
2327       u32 n_left_to_next;
2328
2329       vlib_get_next_frame (vm, node, next_index,
2330                            to_next, n_left_to_next);
2331
2332       while (n_left_from > 0 && n_left_to_next > 0)
2333         {
2334           u32 bi0;
2335           vlib_buffer_t * b0;
2336           u32 next0;
2337           ip4_header_t * ip0;
2338           u32 proto0;
2339           udp_header_t * udp0;
2340           tcp_header_t * tcp0;
2341
2342           /* speculatively enqueue b0 to the current next frame */
2343           bi0 = from[0];
2344           to_next[0] = bi0;
2345           from += 1;
2346           to_next += 1;
2347           n_left_from -= 1;
2348           n_left_to_next -= 1;
2349
2350           b0 = vlib_get_buffer (vm, bi0);
2351           ip0 = vlib_buffer_get_current (b0);
2352           udp0 = ip4_next_header (ip0);
2353           tcp0 = (tcp_header_t *) udp0;
2354
2355           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2356
2357           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2358                                 &next0, 0);
2359
2360           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2361             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2362
2363           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2364
2365           /* verify speculative enqueue, maybe switch current next frame */
2366           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2367                                            to_next, n_left_to_next,
2368                                            bi0, next0);
2369          }
2370
2371       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2372     }
2373
2374   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2375                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2376                                pkts_processed);
2377   return frame->n_vectors;
2378 }
2379
2380 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2381   .function = nat44_hairpinning_fn,
2382   .name = "nat44-hairpinning",
2383   .vector_size = sizeof (u32),
2384   .type = VLIB_NODE_TYPE_INTERNAL,
2385   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2386   .error_strings = snat_in2out_error_strings,
2387   .n_next_nodes = 2,
2388   .next_nodes = {
2389     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2390     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2391   },
2392 };
2393
2394 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2395                               nat44_hairpinning_fn);
2396
2397 static inline void
2398 nat44_reass_hairpinning (snat_main_t *sm,
2399                          vlib_buffer_t * b0,
2400                          ip4_header_t * ip0,
2401                          u16 sport,
2402                          u16 dport,
2403                          u32 proto0)
2404 {
2405   snat_session_key_t key0, sm0;
2406   snat_session_t * s0;
2407   clib_bihash_kv_8_8_t kv0, value0;
2408   ip_csum_t sum0;
2409   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2410   u16 new_dst_port0, old_dst_port0;
2411   udp_header_t * udp0;
2412   tcp_header_t * tcp0;
2413
2414   key0.addr = ip0->dst_address;
2415   key0.port = dport;
2416   key0.protocol = proto0;
2417   key0.fib_index = sm->outside_fib_index;
2418   kv0.key = key0.as_u64;
2419
2420   udp0 = ip4_next_header (ip0);
2421
2422   /* Check if destination is static mappings */
2423   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
2424     {
2425       new_dst_addr0 = sm0.addr.as_u32;
2426       new_dst_port0 = sm0.port;
2427       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2428     }
2429   /* or active sessions */
2430   else
2431     {
2432       if (sm->num_workers > 1)
2433         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2434       else
2435         ti = sm->num_workers;
2436
2437       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2438         {
2439           si = value0.value;
2440           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2441           new_dst_addr0 = s0->in2out.addr.as_u32;
2442           new_dst_port0 = s0->in2out.port;
2443           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2444         }
2445     }
2446
2447   /* Destination is behind the same NAT, use internal address and port */
2448   if (new_dst_addr0)
2449     {
2450       old_dst_addr0 = ip0->dst_address.as_u32;
2451       ip0->dst_address.as_u32 = new_dst_addr0;
2452       sum0 = ip0->checksum;
2453       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2454                              ip4_header_t, dst_address);
2455       ip0->checksum = ip_csum_fold (sum0);
2456
2457       old_dst_port0 = dport;
2458       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2459                        ip4_is_first_fragment (ip0)))
2460         {
2461           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2462             {
2463               tcp0 = ip4_next_header (ip0);
2464               tcp0->dst = new_dst_port0;
2465               sum0 = tcp0->checksum;
2466               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2467                                      ip4_header_t, dst_address);
2468               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2469                                      ip4_header_t /* cheat */, length);
2470               tcp0->checksum = ip_csum_fold(sum0);
2471             }
2472           else
2473             {
2474               udp0->dst_port = new_dst_port0;
2475               udp0->checksum = 0;
2476             }
2477         }
2478       else
2479         {
2480           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2481             {
2482               tcp0 = ip4_next_header (ip0);
2483               sum0 = tcp0->checksum;
2484               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2485                                      ip4_header_t, dst_address);
2486               tcp0->checksum = ip_csum_fold(sum0);
2487             }
2488         }
2489     }
2490 }
2491
2492 static uword
2493 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2494                             vlib_node_runtime_t * node,
2495                             vlib_frame_t * frame)
2496 {
2497   u32 n_left_from, *from, *to_next;
2498   snat_in2out_next_t next_index;
2499   u32 pkts_processed = 0;
2500   snat_main_t *sm = &snat_main;
2501   f64 now = vlib_time_now (vm);
2502   u32 thread_index = vlib_get_thread_index ();
2503   snat_main_per_thread_data_t *per_thread_data =
2504     &sm->per_thread_data[thread_index];
2505   u32 *fragments_to_drop = 0;
2506   u32 *fragments_to_loopback = 0;
2507
2508   from = vlib_frame_vector_args (frame);
2509   n_left_from = frame->n_vectors;
2510   next_index = node->cached_next_index;
2511
2512   while (n_left_from > 0)
2513     {
2514       u32 n_left_to_next;
2515
2516       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2517
2518       while (n_left_from > 0 && n_left_to_next > 0)
2519        {
2520           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2521           vlib_buffer_t *b0;
2522           u32 next0;
2523           u8 cached0 = 0;
2524           ip4_header_t *ip0;
2525           nat_reass_ip4_t *reass0;
2526           udp_header_t * udp0;
2527           tcp_header_t * tcp0;
2528           snat_session_key_t key0;
2529           clib_bihash_kv_8_8_t kv0, value0;
2530           snat_session_t * s0 = 0;
2531           u16 old_port0, new_port0;
2532           ip_csum_t sum0;
2533
2534           /* speculatively enqueue b0 to the current next frame */
2535           bi0 = from[0];
2536           to_next[0] = bi0;
2537           from += 1;
2538           to_next += 1;
2539           n_left_from -= 1;
2540           n_left_to_next -= 1;
2541
2542           b0 = vlib_get_buffer (vm, bi0);
2543           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2544
2545           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2546           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2547                                                                sw_if_index0);
2548
2549           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2550             {
2551               next0 = SNAT_IN2OUT_NEXT_DROP;
2552               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2553               goto trace0;
2554             }
2555
2556           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2557           udp0 = ip4_next_header (ip0);
2558           tcp0 = (tcp_header_t *) udp0;
2559           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2560
2561           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2562                                                  ip0->dst_address,
2563                                                  ip0->fragment_id,
2564                                                  ip0->protocol,
2565                                                  1,
2566                                                  &fragments_to_drop);
2567
2568           if (PREDICT_FALSE (!reass0))
2569             {
2570               next0 = SNAT_IN2OUT_NEXT_DROP;
2571               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2572               goto trace0;
2573             }
2574
2575           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2576             {
2577               key0.addr = ip0->src_address;
2578               key0.port = udp0->src_port;
2579               key0.protocol = proto0;
2580               key0.fib_index = rx_fib_index0;
2581               kv0.key = key0.as_u64;
2582
2583               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2584                 {
2585                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2586                       ip0, proto0, rx_fib_index0, thread_index)))
2587                     goto trace0;
2588
2589                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2590                                      &s0, node, next0, thread_index);
2591
2592                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2593                     goto trace0;
2594
2595                   reass0->sess_index = s0 - per_thread_data->sessions;
2596                 }
2597               else
2598                 {
2599                   s0 = pool_elt_at_index (per_thread_data->sessions,
2600                                           value0.value);
2601                   reass0->sess_index = value0.value;
2602                 }
2603               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2604             }
2605           else
2606             {
2607               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2608                 {
2609                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2610                     {
2611                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2612                       next0 = SNAT_IN2OUT_NEXT_DROP;
2613                       goto trace0;
2614                     }
2615                   cached0 = 1;
2616                   goto trace0;
2617                 }
2618               s0 = pool_elt_at_index (per_thread_data->sessions,
2619                                       reass0->sess_index);
2620             }
2621
2622           old_addr0 = ip0->src_address.as_u32;
2623           ip0->src_address = s0->out2in.addr;
2624           new_addr0 = ip0->src_address.as_u32;
2625           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2626
2627           sum0 = ip0->checksum;
2628           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2629                                  ip4_header_t,
2630                                  src_address /* changed member */);
2631           ip0->checksum = ip_csum_fold (sum0);
2632
2633           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2634             {
2635               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2636                 {
2637                   old_port0 = tcp0->src_port;
2638                   tcp0->src_port = s0->out2in.port;
2639                   new_port0 = tcp0->src_port;
2640
2641                   sum0 = tcp0->checksum;
2642                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2643                                          ip4_header_t,
2644                                          dst_address /* changed member */);
2645                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2646                                          ip4_header_t /* cheat */,
2647                                          length /* changed member */);
2648                   tcp0->checksum = ip_csum_fold(sum0);
2649                 }
2650               else
2651                 {
2652                   old_port0 = udp0->src_port;
2653                   udp0->src_port = s0->out2in.port;
2654                   udp0->checksum = 0;
2655                 }
2656             }
2657
2658           /* Hairpinning */
2659           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2660                                    s0->ext_host_port, proto0);
2661
2662           /* Accounting */
2663           s0->last_heard = now;
2664           s0->total_pkts++;
2665           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2666           /* Per-user LRU list maintenance */
2667           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2668                              s0->per_user_index);
2669           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2670                               s0->per_user_list_head_index,
2671                               s0->per_user_index);
2672
2673         trace0:
2674           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2675                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2676             {
2677               nat44_in2out_reass_trace_t *t =
2678                  vlib_add_trace (vm, node, b0, sizeof (*t));
2679               t->cached = cached0;
2680               t->sw_if_index = sw_if_index0;
2681               t->next_index = next0;
2682             }
2683
2684           if (cached0)
2685             {
2686               n_left_to_next++;
2687               to_next--;
2688             }
2689           else
2690             {
2691               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2692
2693               /* verify speculative enqueue, maybe switch current next frame */
2694               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2695                                                to_next, n_left_to_next,
2696                                                bi0, next0);
2697             }
2698
2699           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2700             {
2701               from = vlib_frame_vector_args (frame);
2702               u32 len = vec_len (fragments_to_loopback);
2703               if (len <= VLIB_FRAME_SIZE)
2704                 {
2705                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2706                   n_left_from = len;
2707                   vec_reset_length (fragments_to_loopback);
2708                 }
2709               else
2710                 {
2711                   clib_memcpy (from,
2712                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2713                                sizeof (u32) * VLIB_FRAME_SIZE);
2714                   n_left_from = VLIB_FRAME_SIZE;
2715                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2716                 }
2717             }
2718        }
2719
2720       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2721     }
2722
2723   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2724                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2725                                pkts_processed);
2726
2727   nat_send_all_to_node (vm, fragments_to_drop, node,
2728                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2729                         SNAT_IN2OUT_NEXT_DROP);
2730
2731   vec_free (fragments_to_drop);
2732   vec_free (fragments_to_loopback);
2733   return frame->n_vectors;
2734 }
2735
2736 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2737   .function = nat44_in2out_reass_node_fn,
2738   .name = "nat44-in2out-reass",
2739   .vector_size = sizeof (u32),
2740   .format_trace = format_nat44_in2out_reass_trace,
2741   .type = VLIB_NODE_TYPE_INTERNAL,
2742
2743   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2744   .error_strings = snat_in2out_error_strings,
2745
2746   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2747   .next_nodes = {
2748     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2749     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2750     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2751     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2752     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2753   },
2754 };
2755
2756 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2757                               nat44_in2out_reass_node_fn);
2758
2759 /**************************/
2760 /*** deterministic mode ***/
2761 /**************************/
2762 static uword
2763 snat_det_in2out_node_fn (vlib_main_t * vm,
2764                          vlib_node_runtime_t * node,
2765                          vlib_frame_t * frame)
2766 {
2767   u32 n_left_from, * from, * to_next;
2768   snat_in2out_next_t next_index;
2769   u32 pkts_processed = 0;
2770   snat_main_t * sm = &snat_main;
2771   u32 now = (u32) vlib_time_now (vm);
2772   u32 thread_index = vlib_get_thread_index ();
2773
2774   from = vlib_frame_vector_args (frame);
2775   n_left_from = frame->n_vectors;
2776   next_index = node->cached_next_index;
2777
2778   while (n_left_from > 0)
2779     {
2780       u32 n_left_to_next;
2781
2782       vlib_get_next_frame (vm, node, next_index,
2783                            to_next, n_left_to_next);
2784
2785       while (n_left_from >= 4 && n_left_to_next >= 2)
2786         {
2787           u32 bi0, bi1;
2788           vlib_buffer_t * b0, * b1;
2789           u32 next0, next1;
2790           u32 sw_if_index0, sw_if_index1;
2791           ip4_header_t * ip0, * ip1;
2792           ip_csum_t sum0, sum1;
2793           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2794           u16 old_port0, new_port0, lo_port0, i0;
2795           u16 old_port1, new_port1, lo_port1, i1;
2796           udp_header_t * udp0, * udp1;
2797           tcp_header_t * tcp0, * tcp1;
2798           u32 proto0, proto1;
2799           snat_det_out_key_t key0, key1;
2800           snat_det_map_t * dm0, * dm1;
2801           snat_det_session_t * ses0 = 0, * ses1 = 0;
2802           u32 rx_fib_index0, rx_fib_index1;
2803           icmp46_header_t * icmp0, * icmp1;
2804
2805           /* Prefetch next iteration. */
2806           {
2807             vlib_buffer_t * p2, * p3;
2808
2809             p2 = vlib_get_buffer (vm, from[2]);
2810             p3 = vlib_get_buffer (vm, from[3]);
2811
2812             vlib_prefetch_buffer_header (p2, LOAD);
2813             vlib_prefetch_buffer_header (p3, LOAD);
2814
2815             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2816             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2817           }
2818
2819           /* speculatively enqueue b0 and b1 to the current next frame */
2820           to_next[0] = bi0 = from[0];
2821           to_next[1] = bi1 = from[1];
2822           from += 2;
2823           to_next += 2;
2824           n_left_from -= 2;
2825           n_left_to_next -= 2;
2826
2827           b0 = vlib_get_buffer (vm, bi0);
2828           b1 = vlib_get_buffer (vm, bi1);
2829
2830           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2831           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2832
2833           ip0 = vlib_buffer_get_current (b0);
2834           udp0 = ip4_next_header (ip0);
2835           tcp0 = (tcp_header_t *) udp0;
2836
2837           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2838
2839           if (PREDICT_FALSE(ip0->ttl == 1))
2840             {
2841               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2842               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2843                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2844                                            0);
2845               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2846               goto trace0;
2847             }
2848
2849           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2850
2851           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2852             {
2853               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2854               icmp0 = (icmp46_header_t *) udp0;
2855
2856               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2857                                   rx_fib_index0, node, next0, thread_index,
2858                                   &ses0, &dm0);
2859               goto trace0;
2860             }
2861
2862           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2863           if (PREDICT_FALSE(!dm0))
2864             {
2865               clib_warning("no match for internal host %U",
2866                            format_ip4_address, &ip0->src_address);
2867               next0 = SNAT_IN2OUT_NEXT_DROP;
2868               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2869               goto trace0;
2870             }
2871
2872           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2873
2874           key0.ext_host_addr = ip0->dst_address;
2875           key0.ext_host_port = tcp0->dst;
2876
2877           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2878           if (PREDICT_FALSE(!ses0))
2879             {
2880               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2881                 {
2882                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2883                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2884
2885                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2886                     continue;
2887
2888                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2889                   break;
2890                 }
2891               if (PREDICT_FALSE(!ses0))
2892                 {
2893                   /* too many sessions for user, send ICMP error packet */
2894
2895                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2896                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2897                                                ICMP4_destination_unreachable_destination_unreachable_host,
2898                                                0);
2899                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2900                   goto trace0;
2901                 }
2902             }
2903
2904           new_port0 = ses0->out.out_port;
2905
2906           old_addr0.as_u32 = ip0->src_address.as_u32;
2907           ip0->src_address.as_u32 = new_addr0.as_u32;
2908           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2909
2910           sum0 = ip0->checksum;
2911           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2912                                  ip4_header_t,
2913                                  src_address /* changed member */);
2914           ip0->checksum = ip_csum_fold (sum0);
2915
2916           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2917             {
2918               if (tcp0->flags & TCP_FLAG_SYN)
2919                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2920               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2921                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2922               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2923                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2924               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2925                 snat_det_ses_close(dm0, ses0);
2926               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2927                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2928               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2929                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2930
2931               old_port0 = tcp0->src;
2932               tcp0->src = new_port0;
2933
2934               sum0 = tcp0->checksum;
2935               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2936                                      ip4_header_t,
2937                                      dst_address /* changed member */);
2938               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2939                                      ip4_header_t /* cheat */,
2940                                      length /* changed member */);
2941               tcp0->checksum = ip_csum_fold(sum0);
2942             }
2943           else
2944             {
2945               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2946               old_port0 = udp0->src_port;
2947               udp0->src_port = new_port0;
2948               udp0->checksum = 0;
2949             }
2950
2951           switch(ses0->state)
2952             {
2953             case SNAT_SESSION_UDP_ACTIVE:
2954                 ses0->expire = now + sm->udp_timeout;
2955                 break;
2956             case SNAT_SESSION_TCP_SYN_SENT:
2957             case SNAT_SESSION_TCP_FIN_WAIT:
2958             case SNAT_SESSION_TCP_CLOSE_WAIT:
2959             case SNAT_SESSION_TCP_LAST_ACK:
2960                 ses0->expire = now + sm->tcp_transitory_timeout;
2961                 break;
2962             case SNAT_SESSION_TCP_ESTABLISHED:
2963                 ses0->expire = now + sm->tcp_established_timeout;
2964                 break;
2965             }
2966
2967         trace0:
2968           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2969                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2970             {
2971               snat_in2out_trace_t *t =
2972                  vlib_add_trace (vm, node, b0, sizeof (*t));
2973               t->is_slow_path = 0;
2974               t->sw_if_index = sw_if_index0;
2975               t->next_index = next0;
2976               t->session_index = ~0;
2977               if (ses0)
2978                 t->session_index = ses0 - dm0->sessions;
2979             }
2980
2981           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2982
2983           ip1 = vlib_buffer_get_current (b1);
2984           udp1 = ip4_next_header (ip1);
2985           tcp1 = (tcp_header_t *) udp1;
2986
2987           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2988
2989           if (PREDICT_FALSE(ip1->ttl == 1))
2990             {
2991               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2992               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2993                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2994                                            0);
2995               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2996               goto trace1;
2997             }
2998
2999           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3000
3001           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3002             {
3003               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3004               icmp1 = (icmp46_header_t *) udp1;
3005
3006               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
3007                                   rx_fib_index1, node, next1, thread_index,
3008                                   &ses1, &dm1);
3009               goto trace1;
3010             }
3011
3012           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
3013           if (PREDICT_FALSE(!dm1))
3014             {
3015               clib_warning("no match for internal host %U",
3016                            format_ip4_address, &ip0->src_address);
3017               next1 = SNAT_IN2OUT_NEXT_DROP;
3018               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3019               goto trace1;
3020             }
3021
3022           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3023
3024           key1.ext_host_addr = ip1->dst_address;
3025           key1.ext_host_port = tcp1->dst;
3026
3027           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3028           if (PREDICT_FALSE(!ses1))
3029             {
3030               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3031                 {
3032                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
3033                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3034
3035                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3036                     continue;
3037
3038                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3039                   break;
3040                 }
3041               if (PREDICT_FALSE(!ses1))
3042                 {
3043                   /* too many sessions for user, send ICMP error packet */
3044
3045                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3046                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3047                                                ICMP4_destination_unreachable_destination_unreachable_host,
3048                                                0);
3049                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3050                   goto trace1;
3051                 }
3052             }
3053
3054           new_port1 = ses1->out.out_port;
3055
3056           old_addr1.as_u32 = ip1->src_address.as_u32;
3057           ip1->src_address.as_u32 = new_addr1.as_u32;
3058           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3059
3060           sum1 = ip1->checksum;
3061           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3062                                  ip4_header_t,
3063                                  src_address /* changed member */);
3064           ip1->checksum = ip_csum_fold (sum1);
3065
3066           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3067             {
3068               if (tcp1->flags & TCP_FLAG_SYN)
3069                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3070               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3071                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3072               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3073                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3074               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3075                 snat_det_ses_close(dm1, ses1);
3076               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3077                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3078               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3079                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3080
3081               old_port1 = tcp1->src;
3082               tcp1->src = new_port1;
3083
3084               sum1 = tcp1->checksum;
3085               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3086                                      ip4_header_t,
3087                                      dst_address /* changed member */);
3088               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3089                                      ip4_header_t /* cheat */,
3090                                      length /* changed member */);
3091               tcp1->checksum = ip_csum_fold(sum1);
3092             }
3093           else
3094             {
3095               ses1->state = SNAT_SESSION_UDP_ACTIVE;
3096               old_port1 = udp1->src_port;
3097               udp1->src_port = new_port1;
3098               udp1->checksum = 0;
3099             }
3100
3101           switch(ses1->state)
3102             {
3103             case SNAT_SESSION_UDP_ACTIVE:
3104                 ses1->expire = now + sm->udp_timeout;
3105                 break;
3106             case SNAT_SESSION_TCP_SYN_SENT:
3107             case SNAT_SESSION_TCP_FIN_WAIT:
3108             case SNAT_SESSION_TCP_CLOSE_WAIT:
3109             case SNAT_SESSION_TCP_LAST_ACK:
3110                 ses1->expire = now + sm->tcp_transitory_timeout;
3111                 break;
3112             case SNAT_SESSION_TCP_ESTABLISHED:
3113                 ses1->expire = now + sm->tcp_established_timeout;
3114                 break;
3115             }
3116
3117         trace1:
3118           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3119                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3120             {
3121               snat_in2out_trace_t *t =
3122                  vlib_add_trace (vm, node, b1, sizeof (*t));
3123               t->is_slow_path = 0;
3124               t->sw_if_index = sw_if_index1;
3125               t->next_index = next1;
3126               t->session_index = ~0;
3127               if (ses1)
3128                 t->session_index = ses1 - dm1->sessions;
3129             }
3130
3131           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3132
3133           /* verify speculative enqueues, maybe switch current next frame */
3134           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3135                                            to_next, n_left_to_next,
3136                                            bi0, bi1, next0, next1);
3137          }
3138
3139       while (n_left_from > 0 && n_left_to_next > 0)
3140         {
3141           u32 bi0;
3142           vlib_buffer_t * b0;
3143           u32 next0;
3144           u32 sw_if_index0;
3145           ip4_header_t * ip0;
3146           ip_csum_t sum0;
3147           ip4_address_t new_addr0, old_addr0;
3148           u16 old_port0, new_port0, lo_port0, i0;
3149           udp_header_t * udp0;
3150           tcp_header_t * tcp0;
3151           u32 proto0;
3152           snat_det_out_key_t key0;
3153           snat_det_map_t * dm0;
3154           snat_det_session_t * ses0 = 0;
3155           u32 rx_fib_index0;
3156           icmp46_header_t * icmp0;
3157
3158           /* speculatively enqueue b0 to the current next frame */
3159           bi0 = from[0];
3160           to_next[0] = bi0;
3161           from += 1;
3162           to_next += 1;
3163           n_left_from -= 1;
3164           n_left_to_next -= 1;
3165
3166           b0 = vlib_get_buffer (vm, bi0);
3167           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3168
3169           ip0 = vlib_buffer_get_current (b0);
3170           udp0 = ip4_next_header (ip0);
3171           tcp0 = (tcp_header_t *) udp0;
3172
3173           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3174
3175           if (PREDICT_FALSE(ip0->ttl == 1))
3176             {
3177               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3178               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3179                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3180                                            0);
3181               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3182               goto trace00;
3183             }
3184
3185           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3186
3187           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3188             {
3189               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3190               icmp0 = (icmp46_header_t *) udp0;
3191
3192               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3193                                   rx_fib_index0, node, next0, thread_index,
3194                                   &ses0, &dm0);
3195               goto trace00;
3196             }
3197
3198           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3199           if (PREDICT_FALSE(!dm0))
3200             {
3201               clib_warning("no match for internal host %U",
3202                            format_ip4_address, &ip0->src_address);
3203               next0 = SNAT_IN2OUT_NEXT_DROP;
3204               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3205               goto trace00;
3206             }
3207
3208           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3209
3210           key0.ext_host_addr = ip0->dst_address;
3211           key0.ext_host_port = tcp0->dst;
3212
3213           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3214           if (PREDICT_FALSE(!ses0))
3215             {
3216               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3217                 {
3218                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3219                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3220
3221                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3222                     continue;
3223
3224                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3225                   break;
3226                 }
3227               if (PREDICT_FALSE(!ses0))
3228                 {
3229                   /* too many sessions for user, send ICMP error packet */
3230
3231                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3232                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3233                                                ICMP4_destination_unreachable_destination_unreachable_host,
3234                                                0);
3235                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3236                   goto trace00;
3237                 }
3238             }
3239
3240           new_port0 = ses0->out.out_port;
3241
3242           old_addr0.as_u32 = ip0->src_address.as_u32;
3243           ip0->src_address.as_u32 = new_addr0.as_u32;
3244           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3245
3246           sum0 = ip0->checksum;
3247           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3248                                  ip4_header_t,
3249                                  src_address /* changed member */);
3250           ip0->checksum = ip_csum_fold (sum0);
3251
3252           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3253             {
3254               if (tcp0->flags & TCP_FLAG_SYN)
3255                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3256               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3257                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3258               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3259                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3260               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3261                 snat_det_ses_close(dm0, ses0);
3262               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3263                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3264               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3265                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3266
3267               old_port0 = tcp0->src;
3268               tcp0->src = new_port0;
3269
3270               sum0 = tcp0->checksum;
3271               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3272                                      ip4_header_t,
3273                                      dst_address /* changed member */);
3274               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3275                                      ip4_header_t /* cheat */,
3276                                      length /* changed member */);
3277               tcp0->checksum = ip_csum_fold(sum0);
3278             }
3279           else
3280             {
3281               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3282               old_port0 = udp0->src_port;
3283               udp0->src_port = new_port0;
3284               udp0->checksum = 0;
3285             }
3286
3287           switch(ses0->state)
3288             {
3289             case SNAT_SESSION_UDP_ACTIVE:
3290                 ses0->expire = now + sm->udp_timeout;
3291                 break;
3292             case SNAT_SESSION_TCP_SYN_SENT:
3293             case SNAT_SESSION_TCP_FIN_WAIT:
3294             case SNAT_SESSION_TCP_CLOSE_WAIT:
3295             case SNAT_SESSION_TCP_LAST_ACK:
3296                 ses0->expire = now + sm->tcp_transitory_timeout;
3297                 break;
3298             case SNAT_SESSION_TCP_ESTABLISHED:
3299                 ses0->expire = now + sm->tcp_established_timeout;
3300                 break;
3301             }
3302
3303         trace00:
3304           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3305                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3306             {
3307               snat_in2out_trace_t *t =
3308                  vlib_add_trace (vm, node, b0, sizeof (*t));
3309               t->is_slow_path = 0;
3310               t->sw_if_index = sw_if_index0;
3311               t->next_index = next0;
3312               t->session_index = ~0;
3313               if (ses0)
3314                 t->session_index = ses0 - dm0->sessions;
3315             }
3316
3317           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3318
3319           /* verify speculative enqueue, maybe switch current next frame */
3320           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3321                                            to_next, n_left_to_next,
3322                                            bi0, next0);
3323         }
3324
3325       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3326     }
3327
3328   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3329                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3330                                pkts_processed);
3331   return frame->n_vectors;
3332 }
3333
3334 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3335   .function = snat_det_in2out_node_fn,
3336   .name = "nat44-det-in2out",
3337   .vector_size = sizeof (u32),
3338   .format_trace = format_snat_in2out_trace,
3339   .type = VLIB_NODE_TYPE_INTERNAL,
3340
3341   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3342   .error_strings = snat_in2out_error_strings,
3343
3344   .runtime_data_bytes = sizeof (snat_runtime_t),
3345
3346   .n_next_nodes = 3,
3347
3348   /* edit / add dispositions here */
3349   .next_nodes = {
3350     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3351     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3352     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3353   },
3354 };
3355
3356 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3357
3358 /**
3359  * Get address and port values to be used for ICMP packet translation
3360  * and create session if needed
3361  *
3362  * @param[in,out] sm             NAT main
3363  * @param[in,out] node           NAT node runtime
3364  * @param[in] thread_index       thread index
3365  * @param[in,out] b0             buffer containing packet to be translated
3366  * @param[out] p_proto           protocol used for matching
3367  * @param[out] p_value           address and port after NAT translation
3368  * @param[out] p_dont_translate  if packet should not be translated
3369  * @param d                      optional parameter
3370  * @param e                      optional parameter
3371  */
3372 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3373                           u32 thread_index, vlib_buffer_t *b0,
3374                           ip4_header_t *ip0, u8 *p_proto,
3375                           snat_session_key_t *p_value,
3376                           u8 *p_dont_translate, void *d, void *e)
3377 {
3378   icmp46_header_t *icmp0;
3379   u32 sw_if_index0;
3380   u32 rx_fib_index0;
3381   u8 protocol;
3382   snat_det_out_key_t key0;
3383   u8 dont_translate = 0;
3384   u32 next0 = ~0;
3385   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3386   ip4_header_t *inner_ip0;
3387   void *l4_header = 0;
3388   icmp46_header_t *inner_icmp0;
3389   snat_det_map_t * dm0 = 0;
3390   ip4_address_t new_addr0;
3391   u16 lo_port0, i0;
3392   snat_det_session_t * ses0 = 0;
3393   ip4_address_t in_addr;
3394   u16 in_port;
3395
3396   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3397   echo0 = (icmp_echo_header_t *)(icmp0+1);
3398   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3399   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3400
3401   if (!icmp_is_error_message (icmp0))
3402     {
3403       protocol = SNAT_PROTOCOL_ICMP;
3404       in_addr = ip0->src_address;
3405       in_port = echo0->identifier;
3406     }
3407   else
3408     {
3409       inner_ip0 = (ip4_header_t *)(echo0+1);
3410       l4_header = ip4_next_header (inner_ip0);
3411       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3412       in_addr = inner_ip0->dst_address;
3413       switch (protocol)
3414         {
3415         case SNAT_PROTOCOL_ICMP:
3416           inner_icmp0 = (icmp46_header_t*)l4_header;
3417           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3418           in_port = inner_echo0->identifier;
3419           break;
3420         case SNAT_PROTOCOL_UDP:
3421         case SNAT_PROTOCOL_TCP:
3422           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3423           break;
3424         default:
3425           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3426           next0 = SNAT_IN2OUT_NEXT_DROP;
3427           goto out;
3428         }
3429     }
3430
3431   dm0 = snat_det_map_by_user(sm, &in_addr);
3432   if (PREDICT_FALSE(!dm0))
3433     {
3434       clib_warning("no match for internal host %U",
3435                    format_ip4_address, &in_addr);
3436       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3437           IP_PROTOCOL_ICMP, rx_fib_index0)))
3438         {
3439           dont_translate = 1;
3440           goto out;
3441         }
3442       next0 = SNAT_IN2OUT_NEXT_DROP;
3443       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3444       goto out;
3445     }
3446
3447   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3448
3449   key0.ext_host_addr = ip0->dst_address;
3450   key0.ext_host_port = 0;
3451
3452   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3453   if (PREDICT_FALSE(!ses0))
3454     {
3455       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3456           IP_PROTOCOL_ICMP, rx_fib_index0)))
3457         {
3458           dont_translate = 1;
3459           goto out;
3460         }
3461       if (icmp0->type != ICMP4_echo_request)
3462         {
3463           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3464           next0 = SNAT_IN2OUT_NEXT_DROP;
3465           goto out;
3466         }
3467       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3468         {
3469           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3470             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3471
3472           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3473             continue;
3474
3475           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3476           break;
3477         }
3478       if (PREDICT_FALSE(!ses0))
3479         {
3480           next0 = SNAT_IN2OUT_NEXT_DROP;
3481           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3482           goto out;
3483         }
3484     }
3485
3486   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3487                     !icmp_is_error_message (icmp0)))
3488     {
3489       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3490       next0 = SNAT_IN2OUT_NEXT_DROP;
3491       goto out;
3492     }
3493
3494   u32 now = (u32) vlib_time_now (sm->vlib_main);
3495
3496   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3497   ses0->expire = now + sm->icmp_timeout;
3498
3499 out:
3500   *p_proto = protocol;
3501   if (ses0)
3502     {
3503       p_value->addr = new_addr0;
3504       p_value->fib_index = sm->outside_fib_index;
3505       p_value->port = ses0->out.out_port;
3506     }
3507   *p_dont_translate = dont_translate;
3508   if (d)
3509     *(snat_det_session_t**)d = ses0;
3510   if (e)
3511     *(snat_det_map_t**)e = dm0;
3512   return next0;
3513 }
3514
3515 /**********************/
3516 /*** worker handoff ***/
3517 /**********************/
3518 static inline uword
3519 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3520                                       vlib_node_runtime_t * node,
3521                                       vlib_frame_t * frame,
3522                                       u8 is_output)
3523 {
3524   snat_main_t *sm = &snat_main;
3525   vlib_thread_main_t *tm = vlib_get_thread_main ();
3526   u32 n_left_from, *from, *to_next = 0;
3527   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3528   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3529     = 0;
3530   vlib_frame_queue_elt_t *hf = 0;
3531   vlib_frame_t *f = 0;
3532   int i;
3533   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3534   u32 next_worker_index = 0;
3535   u32 current_worker_index = ~0;
3536   u32 thread_index = vlib_get_thread_index ();
3537   u32 fq_index;
3538   u32 to_node_index;
3539
3540   ASSERT (vec_len (sm->workers));
3541
3542   if (is_output)
3543     {
3544       fq_index = sm->fq_in2out_output_index;
3545       to_node_index = sm->in2out_output_node_index;
3546     }
3547   else
3548     {
3549       fq_index = sm->fq_in2out_index;
3550       to_node_index = sm->in2out_node_index;
3551     }
3552
3553   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3554     {
3555       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3556
3557       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3558                                sm->first_worker_index + sm->num_workers - 1,
3559                                (vlib_frame_queue_t *) (~0));
3560     }
3561
3562   from = vlib_frame_vector_args (frame);
3563   n_left_from = frame->n_vectors;
3564
3565   while (n_left_from > 0)
3566     {
3567       u32 bi0;
3568       vlib_buffer_t *b0;
3569       u32 sw_if_index0;
3570       u32 rx_fib_index0;
3571       ip4_header_t * ip0;
3572       u8 do_handoff;
3573
3574       bi0 = from[0];
3575       from += 1;
3576       n_left_from -= 1;
3577
3578       b0 = vlib_get_buffer (vm, bi0);
3579
3580       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3581       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3582
3583       ip0 = vlib_buffer_get_current (b0);
3584
3585       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3586
3587       if (PREDICT_FALSE (next_worker_index != thread_index))
3588         {
3589           do_handoff = 1;
3590
3591           if (next_worker_index != current_worker_index)
3592             {
3593               if (hf)
3594                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3595
3596               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3597                                                       next_worker_index,
3598                                                       handoff_queue_elt_by_worker_index);
3599
3600               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3601               to_next_worker = &hf->buffer_index[hf->n_vectors];
3602               current_worker_index = next_worker_index;
3603             }
3604
3605           /* enqueue to correct worker thread */
3606           to_next_worker[0] = bi0;
3607           to_next_worker++;
3608           n_left_to_next_worker--;
3609
3610           if (n_left_to_next_worker == 0)
3611             {
3612               hf->n_vectors = VLIB_FRAME_SIZE;
3613               vlib_put_frame_queue_elt (hf);
3614               current_worker_index = ~0;
3615               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3616               hf = 0;
3617             }
3618         }
3619       else
3620         {
3621           do_handoff = 0;
3622           /* if this is 1st frame */
3623           if (!f)
3624             {
3625               f = vlib_get_frame_to_node (vm, to_node_index);
3626               to_next = vlib_frame_vector_args (f);
3627             }
3628
3629           to_next[0] = bi0;
3630           to_next += 1;
3631           f->n_vectors++;
3632         }
3633
3634       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3635                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3636         {
3637           snat_in2out_worker_handoff_trace_t *t =
3638             vlib_add_trace (vm, node, b0, sizeof (*t));
3639           t->next_worker_index = next_worker_index;
3640           t->do_handoff = do_handoff;
3641         }
3642     }
3643
3644   if (f)
3645     vlib_put_frame_to_node (vm, to_node_index, f);
3646
3647   if (hf)
3648     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3649
3650   /* Ship frames to the worker nodes */
3651   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3652     {
3653       if (handoff_queue_elt_by_worker_index[i])
3654         {
3655           hf = handoff_queue_elt_by_worker_index[i];
3656           /*
3657            * It works better to let the handoff node
3658            * rate-adapt, always ship the handoff queue element.
3659            */
3660           if (1 || hf->n_vectors == hf->last_n_vectors)
3661             {
3662               vlib_put_frame_queue_elt (hf);
3663               handoff_queue_elt_by_worker_index[i] = 0;
3664             }
3665           else
3666             hf->last_n_vectors = hf->n_vectors;
3667         }
3668       congested_handoff_queue_by_worker_index[i] =
3669         (vlib_frame_queue_t *) (~0);
3670     }
3671   hf = 0;
3672   current_worker_index = ~0;
3673   return frame->n_vectors;
3674 }
3675
3676 static uword
3677 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3678                                vlib_node_runtime_t * node,
3679                                vlib_frame_t * frame)
3680 {
3681   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3682 }
3683
3684 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3685   .function = snat_in2out_worker_handoff_fn,
3686   .name = "nat44-in2out-worker-handoff",
3687   .vector_size = sizeof (u32),
3688   .format_trace = format_snat_in2out_worker_handoff_trace,
3689   .type = VLIB_NODE_TYPE_INTERNAL,
3690
3691   .n_next_nodes = 1,
3692
3693   .next_nodes = {
3694     [0] = "error-drop",
3695   },
3696 };
3697
3698 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3699                               snat_in2out_worker_handoff_fn);
3700
3701 static uword
3702 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3703                                       vlib_node_runtime_t * node,
3704                                       vlib_frame_t * frame)
3705 {
3706   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3707 }
3708
3709 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3710   .function = snat_in2out_output_worker_handoff_fn,
3711   .name = "nat44-in2out-output-worker-handoff",
3712   .vector_size = sizeof (u32),
3713   .format_trace = format_snat_in2out_worker_handoff_trace,
3714   .type = VLIB_NODE_TYPE_INTERNAL,
3715
3716   .n_next_nodes = 1,
3717
3718   .next_nodes = {
3719     [0] = "error-drop",
3720   },
3721 };
3722
3723 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3724                               snat_in2out_output_worker_handoff_fn);
3725
3726 static_always_inline int
3727 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3728 {
3729   snat_address_t * ap;
3730   clib_bihash_kv_8_8_t kv, value;
3731   snat_session_key_t m_key;
3732
3733   vec_foreach (ap, sm->addresses)
3734     {
3735       if (ap->addr.as_u32 == dst_addr->as_u32)
3736         return 1;
3737     }
3738
3739   m_key.addr.as_u32 = dst_addr->as_u32;
3740   m_key.fib_index = sm->outside_fib_index;
3741   m_key.port = 0;
3742   m_key.protocol = 0;
3743   kv.key = m_key.as_u64;
3744   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3745     return 1;
3746
3747   return 0;
3748 }
3749
3750 static uword
3751 snat_hairpin_dst_fn (vlib_main_t * vm,
3752                      vlib_node_runtime_t * node,
3753                      vlib_frame_t * frame)
3754 {
3755   u32 n_left_from, * from, * to_next;
3756   snat_in2out_next_t next_index;
3757   u32 pkts_processed = 0;
3758   snat_main_t * sm = &snat_main;
3759
3760   from = vlib_frame_vector_args (frame);
3761   n_left_from = frame->n_vectors;
3762   next_index = node->cached_next_index;
3763
3764   while (n_left_from > 0)
3765     {
3766       u32 n_left_to_next;
3767
3768       vlib_get_next_frame (vm, node, next_index,
3769                            to_next, n_left_to_next);
3770
3771       while (n_left_from > 0 && n_left_to_next > 0)
3772         {
3773           u32 bi0;
3774           vlib_buffer_t * b0;
3775           u32 next0;
3776           ip4_header_t * ip0;
3777           u32 proto0;
3778
3779           /* speculatively enqueue b0 to the current next frame */
3780           bi0 = from[0];
3781           to_next[0] = bi0;
3782           from += 1;
3783           to_next += 1;
3784           n_left_from -= 1;
3785           n_left_to_next -= 1;
3786
3787           b0 = vlib_get_buffer (vm, bi0);
3788           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3789           ip0 = vlib_buffer_get_current (b0);
3790
3791           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3792
3793           vnet_buffer (b0)->snat.flags = 0;
3794           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3795             {
3796               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3797                 {
3798                   udp_header_t * udp0 = ip4_next_header (ip0);
3799                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3800
3801                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3802                 }
3803               else if (proto0 == SNAT_PROTOCOL_ICMP)
3804                 {
3805                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3806
3807                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3808                 }
3809               else
3810                 {
3811                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3812                 }
3813
3814               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3815             }
3816
3817           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3818
3819           /* verify speculative enqueue, maybe switch current next frame */
3820           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3821                                            to_next, n_left_to_next,
3822                                            bi0, next0);
3823          }
3824
3825       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3826     }
3827
3828   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3829                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3830                                pkts_processed);
3831   return frame->n_vectors;
3832 }
3833
3834 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3835   .function = snat_hairpin_dst_fn,
3836   .name = "nat44-hairpin-dst",
3837   .vector_size = sizeof (u32),
3838   .type = VLIB_NODE_TYPE_INTERNAL,
3839   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3840   .error_strings = snat_in2out_error_strings,
3841   .n_next_nodes = 2,
3842   .next_nodes = {
3843     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3844     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3845   },
3846 };
3847
3848 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3849                               snat_hairpin_dst_fn);
3850
3851 static uword
3852 snat_hairpin_src_fn (vlib_main_t * vm,
3853                      vlib_node_runtime_t * node,
3854                      vlib_frame_t * frame)
3855 {
3856   u32 n_left_from, * from, * to_next;
3857   snat_in2out_next_t next_index;
3858   u32 pkts_processed = 0;
3859   snat_main_t *sm = &snat_main;
3860
3861   from = vlib_frame_vector_args (frame);
3862   n_left_from = frame->n_vectors;
3863   next_index = node->cached_next_index;
3864
3865   while (n_left_from > 0)
3866     {
3867       u32 n_left_to_next;
3868
3869       vlib_get_next_frame (vm, node, next_index,
3870                            to_next, n_left_to_next);
3871
3872       while (n_left_from > 0 && n_left_to_next > 0)
3873         {
3874           u32 bi0;
3875           vlib_buffer_t * b0;
3876           u32 next0;
3877           snat_interface_t *i;
3878           u32 sw_if_index0;
3879
3880           /* speculatively enqueue b0 to the current next frame */
3881           bi0 = from[0];
3882           to_next[0] = bi0;
3883           from += 1;
3884           to_next += 1;
3885           n_left_from -= 1;
3886           n_left_to_next -= 1;
3887
3888           b0 = vlib_get_buffer (vm, bi0);
3889           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3890           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3891
3892           pool_foreach (i, sm->output_feature_interfaces,
3893           ({
3894             /* Only packets from NAT inside interface */
3895             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3896               {
3897                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3898                                     SNAT_FLAG_HAIRPINNING))
3899                   {
3900                     if (PREDICT_TRUE (sm->num_workers > 1))
3901                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3902                     else
3903                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3904                   }
3905                 break;
3906               }
3907           }));
3908
3909           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3910
3911           /* verify speculative enqueue, maybe switch current next frame */
3912           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3913                                            to_next, n_left_to_next,
3914                                            bi0, next0);
3915          }
3916
3917       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3918     }
3919
3920   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3921                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3922                                pkts_processed);
3923   return frame->n_vectors;
3924 }
3925
3926 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3927   .function = snat_hairpin_src_fn,
3928   .name = "nat44-hairpin-src",
3929   .vector_size = sizeof (u32),
3930   .type = VLIB_NODE_TYPE_INTERNAL,
3931   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3932   .error_strings = snat_in2out_error_strings,
3933   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3934   .next_nodes = {
3935      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3936      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3937      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3938      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3939   },
3940 };
3941
3942 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3943                               snat_hairpin_src_fn);
3944
3945 static uword
3946 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3947                                 vlib_node_runtime_t * node,
3948                                 vlib_frame_t * frame)
3949 {
3950   u32 n_left_from, * from, * to_next;
3951   snat_in2out_next_t next_index;
3952   u32 pkts_processed = 0;
3953   snat_main_t * sm = &snat_main;
3954   u32 stats_node_index;
3955
3956   stats_node_index = snat_in2out_fast_node.index;
3957
3958   from = vlib_frame_vector_args (frame);
3959   n_left_from = frame->n_vectors;
3960   next_index = node->cached_next_index;
3961
3962   while (n_left_from > 0)
3963     {
3964       u32 n_left_to_next;
3965
3966       vlib_get_next_frame (vm, node, next_index,
3967                            to_next, n_left_to_next);
3968
3969       while (n_left_from > 0 && n_left_to_next > 0)
3970         {
3971           u32 bi0;
3972           vlib_buffer_t * b0;
3973           u32 next0;
3974           u32 sw_if_index0;
3975           ip4_header_t * ip0;
3976           ip_csum_t sum0;
3977           u32 new_addr0, old_addr0;
3978           u16 old_port0, new_port0;
3979           udp_header_t * udp0;
3980           tcp_header_t * tcp0;
3981           icmp46_header_t * icmp0;
3982           snat_session_key_t key0, sm0;
3983           u32 proto0;
3984           u32 rx_fib_index0;
3985
3986           /* speculatively enqueue b0 to the current next frame */
3987           bi0 = from[0];
3988           to_next[0] = bi0;
3989           from += 1;
3990           to_next += 1;
3991           n_left_from -= 1;
3992           n_left_to_next -= 1;
3993
3994           b0 = vlib_get_buffer (vm, bi0);
3995           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3996
3997           ip0 = vlib_buffer_get_current (b0);
3998           udp0 = ip4_next_header (ip0);
3999           tcp0 = (tcp_header_t *) udp0;
4000           icmp0 = (icmp46_header_t *) udp0;
4001
4002           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4003           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4004
4005           if (PREDICT_FALSE(ip0->ttl == 1))
4006             {
4007               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4008               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4009                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4010                                            0);
4011               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4012               goto trace0;
4013             }
4014
4015           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4016
4017           if (PREDICT_FALSE (proto0 == ~0))
4018               goto trace0;
4019
4020           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4021             {
4022               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4023                                   rx_fib_index0, node, next0, ~0, 0, 0);
4024               goto trace0;
4025             }
4026
4027           key0.addr = ip0->src_address;
4028           key0.protocol = proto0;
4029           key0.port = udp0->src_port;
4030           key0.fib_index = rx_fib_index0;
4031
4032           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0))
4033             {
4034               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4035               next0= SNAT_IN2OUT_NEXT_DROP;
4036               goto trace0;
4037             }
4038
4039           new_addr0 = sm0.addr.as_u32;
4040           new_port0 = sm0.port;
4041           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4042           old_addr0 = ip0->src_address.as_u32;
4043           ip0->src_address.as_u32 = new_addr0;
4044
4045           sum0 = ip0->checksum;
4046           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4047                                  ip4_header_t,
4048                                  src_address /* changed member */);
4049           ip0->checksum = ip_csum_fold (sum0);
4050
4051           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4052             {
4053               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4054                 {
4055                   old_port0 = tcp0->src_port;
4056                   tcp0->src_port = new_port0;
4057
4058                   sum0 = tcp0->checksum;
4059                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4060                                          ip4_header_t,
4061                                          dst_address /* changed member */);
4062                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4063                                          ip4_header_t /* cheat */,
4064                                          length /* changed member */);
4065                   tcp0->checksum = ip_csum_fold(sum0);
4066                 }
4067               else
4068                 {
4069                   old_port0 = udp0->src_port;
4070                   udp0->src_port = new_port0;
4071                   udp0->checksum = 0;
4072                 }
4073             }
4074           else
4075             {
4076               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4077                 {
4078                   sum0 = tcp0->checksum;
4079                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4080                                          ip4_header_t,
4081                                          dst_address /* changed member */);
4082                   tcp0->checksum = ip_csum_fold(sum0);
4083                 }
4084             }
4085
4086           /* Hairpinning */
4087           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4088
4089         trace0:
4090           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4091                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4092             {
4093               snat_in2out_trace_t *t =
4094                  vlib_add_trace (vm, node, b0, sizeof (*t));
4095               t->sw_if_index = sw_if_index0;
4096               t->next_index = next0;
4097             }
4098
4099           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4100
4101           /* verify speculative enqueue, maybe switch current next frame */
4102           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4103                                            to_next, n_left_to_next,
4104                                            bi0, next0);
4105         }
4106
4107       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4108     }
4109
4110   vlib_node_increment_counter (vm, stats_node_index,
4111                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4112                                pkts_processed);
4113   return frame->n_vectors;
4114 }
4115
4116
4117 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
4118   .function = snat_in2out_fast_static_map_fn,
4119   .name = "nat44-in2out-fast",
4120   .vector_size = sizeof (u32),
4121   .format_trace = format_snat_in2out_fast_trace,
4122   .type = VLIB_NODE_TYPE_INTERNAL,
4123
4124   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4125   .error_strings = snat_in2out_error_strings,
4126
4127   .runtime_data_bytes = sizeof (snat_runtime_t),
4128
4129   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4130
4131   /* edit / add dispositions here */
4132   .next_nodes = {
4133     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4134     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4135     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4136     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4137     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4138   },
4139 };
4140
4141 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);