src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37   u32 is_slow_path;
  38 } snat_in2out_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_in2out_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  51   char * tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  66
  67   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  68               t->sw_if_index, t->next_index);
  69
  70   return s;
  71 }
  72
  73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   snat_in2out_worker_handoff_trace_t * t =
  78     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  79   char * m;
  80
  81   m = t->do_handoff ? "next worker" : "same worker";
  82   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  83
  84   return s;
  85 }
  86
  87 typedef struct {
  88   u32 sw_if_index;
  89   u32 next_index;
  90   u8 cached;
  91 } nat44_in2out_reass_trace_t;
  92
  93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  94 {
  95   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  96   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  97   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  98
  99   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 100               t->sw_if_index, t->next_index,
 101               t->cached ? "cached" : "translated");
 102
 103   return s;
 104 }
 105
 106 vlib_node_registration_t snat_in2out_node;
 107 vlib_node_registration_t snat_in2out_slowpath_node;
 108 vlib_node_registration_t snat_in2out_fast_node;
 109 vlib_node_registration_t snat_in2out_worker_handoff_node;
 110 vlib_node_registration_t snat_det_in2out_node;
 111 vlib_node_registration_t snat_in2out_output_node;
 112 vlib_node_registration_t snat_in2out_output_slowpath_node;
 113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 114 vlib_node_registration_t snat_hairpin_dst_node;
 115 vlib_node_registration_t snat_hairpin_src_node;
 116 vlib_node_registration_t nat44_hairpinning_node;
 117 vlib_node_registration_t nat44_in2out_reass_node;
 118
 119
 120 #define foreach_snat_in2out_error                       \
 121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 122 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 123 _(OUT_OF_PORTS, "Out of ports")                         \
 124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 125 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 126 _(NO_TRANSLATION, "No translation")                     \
 127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 128 _(DROP_FRAGMENT, "Drop fragment")                       \
 129 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 131
 132 typedef enum {
 133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 134   foreach_snat_in2out_error
 135 #undef _
 136   SNAT_IN2OUT_N_ERROR,
 137 } snat_in2out_error_t;
 138
 139 static char * snat_in2out_error_strings[] = {
 140 #define _(sym,string) string,
 141   foreach_snat_in2out_error
 142 #undef _
 143 };
 144
 145 typedef enum {
 146   SNAT_IN2OUT_NEXT_LOOKUP,
 147   SNAT_IN2OUT_NEXT_DROP,
 148   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 149   SNAT_IN2OUT_NEXT_SLOW_PATH,
 150   SNAT_IN2OUT_NEXT_REASS,
 151   SNAT_IN2OUT_N_NEXT,
 152 } snat_in2out_next_t;
 153
 154 typedef enum {
 155   SNAT_HAIRPIN_SRC_NEXT_DROP,
 156   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 157   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 158   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 159   SNAT_HAIRPIN_SRC_N_NEXT,
 160 } snat_hairpin_next_t;
 161
 162 /**
 163  * @brief Check if packet should be translated
 164  *
 165  * Packets aimed at outside interface and external address with active session
 166  * should be translated.
 167  *
 168  * @param sm            NAT main
 169  * @param rt            NAT runtime data
 170  * @param sw_if_index0  index of the inside interface
 171  * @param ip0           IPv4 header
 172  * @param proto0        NAT protocol
 173  * @param rx_fib_index0 RX FIB index
 174  *
 175  * @returns 0 if packet should be translated otherwise 1
 176  */
 177 static inline int
 178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 179                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                          u32 rx_fib_index0)
 181 {
 182   if (sm->out2in_dpo)
 183     return 0;
 184
 185   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 186   fib_prefix_t pfx = {
 187     .fp_proto = FIB_PROTOCOL_IP4,
 188     .fp_len = 32,
 189     .fp_addr = {
 190         .ip4.as_u32 = ip0->dst_address.as_u32,
 191     },
 192   };
 193
 194   /* Don't NAT packet aimed at the intfc address */
 195   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 196                                       ip0->dst_address.as_u32)))
 197     return 1;
 198
 199   fei = fib_table_lookup (rx_fib_index0, &pfx);
 200   if (FIB_NODE_INDEX_INVALID != fei)
 201     {
 202       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 203       if (sw_if_index == ~0)
 204         {
 205           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 206           if (FIB_NODE_INDEX_INVALID != fei)
 207             sw_if_index = fib_entry_get_resolving_interface (fei);
 208         }
 209       snat_interface_t *i;
 210       pool_foreach (i, sm->interfaces,
 211       ({
 212         /* NAT packet aimed at outside interface */
 213         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 214           return 0;
 215       }));
 216     }
 217
 218   return 1;
 219 }
 220
 221 static inline int
 222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 223                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 224                     u32 rx_fib_index0, u32 thread_index)
 225 {
 226   udp_header_t * udp0 = ip4_next_header (ip0);
 227   snat_session_key_t key0, sm0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229
 230   key0.addr = ip0->dst_address;
 231   key0.port = udp0->dst_port;
 232   key0.protocol = proto0;
 233   key0.fib_index = sm->outside_fib_index;
 234   kv0.key = key0.as_u64;
 235
 236   /* NAT packet aimed at external address if */
 237   /* has active sessions */
 238   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 239                               &value0))
 240     {
 241       /* or is static mappings */
 242       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 243         return 0;
 244     }
 245   else
 246     return 0;
 247
 248   if (sm->forwarding_enabled)
 249     return 1;
 250
 251   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 252                                  rx_fib_index0);
 253 }
 254
 255 static inline int
 256 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 257                                   u32 proto0, u16 src_port, u16 dst_port,
 258                                   u32 thread_index, u32 sw_if_index)
 259 {
 260   snat_session_key_t key0;
 261   clib_bihash_kv_8_8_t kv0, value0;
 262   snat_interface_t *i;
 263
 264   /* src NAT check */
 265   key0.addr = ip0->src_address;
 266   key0.port = src_port;
 267   key0.protocol = proto0;
 268   key0.fib_index = sm->outside_fib_index;
 269   kv0.key = key0.as_u64;
 270
 271   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 272                                &value0))
 273     return 1;
 274
 275   /* dst NAT check */
 276   key0.addr = ip0->dst_address;
 277   key0.port = dst_port;
 278   key0.protocol = proto0;
 279   key0.fib_index = sm->inside_fib_index;
 280   kv0.key = key0.as_u64;
 281   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 282                                &value0))
 283   {
 284     /* hairpinning */
 285     pool_foreach (i, sm->output_feature_interfaces,
 286     ({
 287       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 288         return 0;
 289     }));
 290     return 1;
 291   }
 292
 293   return 0;
 294 }
 295
 296 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 297                       ip4_header_t * ip0,
 298                       u32 rx_fib_index0,
 299                       snat_session_key_t * key0,
 300                       snat_session_t ** sessionp,
 301                       vlib_node_runtime_t * node,
 302                       u32 next0,
 303                       u32 thread_index)
 304 {
 305   snat_user_t *u;
 306   snat_session_t *s;
 307   clib_bihash_kv_8_8_t kv0;
 308   snat_session_key_t key1;
 309   u32 address_index = ~0;
 310   u32 outside_fib_index;
 311   uword * p;
 312   udp_header_t * udp0 = ip4_next_header (ip0);
 313   u8 is_sm = 0;
 314
 315   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 316     {
 317       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 318       nat_ipfix_logging_max_sessions(sm->max_translations);
 319       return SNAT_IN2OUT_NEXT_DROP;
 320     }
 321
 322   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 323   if (! p)
 324     {
 325       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 326       return SNAT_IN2OUT_NEXT_DROP;
 327     }
 328   outside_fib_index = p[0];
 329
 330   key1.protocol = key0->protocol;
 331
 332   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 333                               thread_index);
 334   if (!u)
 335     {
 336       clib_warning ("create NAT user failed");
 337       return SNAT_IN2OUT_NEXT_DROP;
 338     }
 339
 340   /* First try to match static mapping by local address and port */
 341   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
 342     {
 343       /* Try to create dynamic translation */
 344       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 345                                                thread_index, &key1,
 346                                                &address_index,
 347                                                sm->port_per_thread,
 348                                                sm->per_thread_data[thread_index].snat_thread_index))
 349         {
 350           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 351           return SNAT_IN2OUT_NEXT_DROP;
 352         }
 353     }
 354   else
 355     is_sm = 1;
 356
 357   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 358   if (!s)
 359     {
 360       clib_warning ("create NAT session failed");
 361       return SNAT_IN2OUT_NEXT_DROP;
 362     }
 363
 364   if (is_sm)
 365     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 366   user_session_increment (sm, u, is_sm);
 367   s->outside_address_index = address_index;
 368   s->in2out = *key0;
 369   s->out2in = key1;
 370   s->out2in.protocol = key0->protocol;
 371   s->out2in.fib_index = outside_fib_index;
 372   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 373   s->ext_host_port = udp0->dst_port;
 374   *sessionp = s;
 375
 376   /* Add to translation hashes */
 377   kv0.key = s->in2out.as_u64;
 378   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 379   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 380                                1 /* is_add */))
 381       clib_warning ("in2out key add failed");
 382
 383   kv0.key = s->out2in.as_u64;
 384   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 385
 386   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 387                                1 /* is_add */))
 388       clib_warning ("out2in key add failed");
 389
 390   /* log NAT event */
 391   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 392                                       s->out2in.addr.as_u32,
 393                                       s->in2out.protocol,
 394                                       s->in2out.port,
 395                                       s->out2in.port,
 396                                       s->in2out.fib_index);
 397   return next0;
 398 }
 399
 400 static_always_inline
 401 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 402                                  snat_session_key_t *p_key0)
 403 {
 404   icmp46_header_t *icmp0;
 405   snat_session_key_t key0;
 406   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 407   ip4_header_t *inner_ip0 = 0;
 408   void *l4_header = 0;
 409   icmp46_header_t *inner_icmp0;
 410
 411   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 412   echo0 = (icmp_echo_header_t *)(icmp0+1);
 413
 414   if (!icmp_is_error_message (icmp0))
 415     {
 416       key0.protocol = SNAT_PROTOCOL_ICMP;
 417       key0.addr = ip0->src_address;
 418       key0.port = echo0->identifier;
 419     }
 420   else
 421     {
 422       inner_ip0 = (ip4_header_t *)(echo0+1);
 423       l4_header = ip4_next_header (inner_ip0);
 424       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 425       key0.addr = inner_ip0->dst_address;
 426       switch (key0.protocol)
 427         {
 428         case SNAT_PROTOCOL_ICMP:
 429           inner_icmp0 = (icmp46_header_t*)l4_header;
 430           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 431           key0.port = inner_echo0->identifier;
 432           break;
 433         case SNAT_PROTOCOL_UDP:
 434         case SNAT_PROTOCOL_TCP:
 435           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 436           break;
 437         default:
 438           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 439         }
 440     }
 441   *p_key0 = key0;
 442   return -1; /* success */
 443 }
 444
 445 static_always_inline int
 446 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 447 {
 448   icmp46_header_t *icmp0;
 449   nat_ed_ses_key_t key0;
 450   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 451   ip4_header_t *inner_ip0 = 0;
 452   void *l4_header = 0;
 453   icmp46_header_t *inner_icmp0;
 454
 455   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 456   echo0 = (icmp_echo_header_t *)(icmp0+1);
 457
 458   if (!icmp_is_error_message (icmp0))
 459     {
 460       key0.proto = IP_PROTOCOL_ICMP;
 461       key0.l_addr = ip0->src_address;
 462       key0.r_addr = ip0->dst_address;
 463       key0.l_port = key0.r_port = echo0->identifier;
 464     }
 465   else
 466     {
 467       inner_ip0 = (ip4_header_t *)(echo0+1);
 468       l4_header = ip4_next_header (inner_ip0);
 469       key0.proto = inner_ip0->protocol;
 470       key0.r_addr = inner_ip0->src_address;
 471       key0.l_addr = inner_ip0->dst_address;
 472       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 473         {
 474         case SNAT_PROTOCOL_ICMP:
 475           inner_icmp0 = (icmp46_header_t*)l4_header;
 476           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 477           key0.r_port = key0.l_port = inner_echo0->identifier;
 478           break;
 479         case SNAT_PROTOCOL_UDP:
 480         case SNAT_PROTOCOL_TCP:
 481           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 482           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
 483           break;
 484         default:
 485           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 486         }
 487     }
 488   *p_key0 = key0;
 489   return 0;
 490 }
 491
 492 static inline int
 493 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip)
 494 {
 495   nat_ed_ses_key_t key;
 496   clib_bihash_kv_16_8_t kv, value;
 497   udp_header_t *udp;
 498
 499   if (!sm->forwarding_enabled)
 500     return 0;
 501
 502   if (ip->protocol == IP_PROTOCOL_ICMP)
 503     {
 504       if (icmp_get_ed_key (ip, &key))
 505         return 0;
 506     }
 507   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 508     {
 509       udp = ip4_next_header(ip);
 510       key.l_addr = ip->src_address;
 511       key.r_addr = ip->dst_address;
 512       key.proto = ip->protocol;
 513       key.r_port = udp->dst_port;
 514       key.l_port = udp->src_port;
 515     }
 516   else
 517     {
 518       key.l_addr = ip->src_address;
 519       key.r_addr = ip->dst_address;
 520       key.proto = ip->protocol;
 521       key.l_port = key.r_port = 0;
 522     }
 523   key.fib_index = 0;
 524   kv.key[0] = key.as_u64[0];
 525   kv.key[1] = key.as_u64[1];
 526
 527   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 528     return value.value == ~0ULL;
 529
 530   return 0;
 531 }
 532
 533 /**
 534  * Get address and port values to be used for ICMP packet translation
 535  * and create session if needed
 536  *
 537  * @param[in,out] sm             NAT main
 538  * @param[in,out] node           NAT node runtime
 539  * @param[in] thread_index       thread index
 540  * @param[in,out] b0             buffer containing packet to be translated
 541  * @param[out] p_proto           protocol used for matching
 542  * @param[out] p_value           address and port after NAT translation
 543  * @param[out] p_dont_translate  if packet should not be translated
 544  * @param d                      optional parameter
 545  * @param e                      optional parameter
 546  */
 547 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 548                            u32 thread_index, vlib_buffer_t *b0,
 549                            ip4_header_t *ip0, u8 *p_proto,
 550                            snat_session_key_t *p_value,
 551                            u8 *p_dont_translate, void *d, void *e)
 552 {
 553   icmp46_header_t *icmp0;
 554   u32 sw_if_index0;
 555   u32 rx_fib_index0;
 556   snat_session_key_t key0;
 557   snat_session_t *s0 = 0;
 558   u8 dont_translate = 0;
 559   clib_bihash_kv_8_8_t kv0, value0;
 560   u32 next0 = ~0;
 561   int err;
 562
 563   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 564   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 565   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 566
 567   err = icmp_get_key (ip0, &key0);
 568   if (err != -1)
 569     {
 570       b0->error = node->errors[err];
 571       next0 = SNAT_IN2OUT_NEXT_DROP;
 572       goto out;
 573     }
 574   key0.fib_index = rx_fib_index0;
 575
 576   kv0.key = key0.as_u64;
 577
 578   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 579                               &value0))
 580     {
 581       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 582         {
 583           if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
 584               ip0, SNAT_PROTOCOL_ICMP, key0.port, key0.port, thread_index, sw_if_index0)))
 585             {
 586               dont_translate = 1;
 587               goto out;
 588             }
 589         }
 590       else
 591         {
 592           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 593               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 594             {
 595               dont_translate = 1;
 596               goto out;
 597             }
 598         }
 599
 600       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 601         {
 602           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 603           next0 = SNAT_IN2OUT_NEXT_DROP;
 604           goto out;
 605         }
 606
 607       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 608                          &s0, node, next0, thread_index);
 609
 610       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 611         goto out;
 612     }
 613   else
 614     {
 615       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 616                         icmp0->type != ICMP4_echo_reply &&
 617                         !icmp_is_error_message (icmp0)))
 618         {
 619           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 620           next0 = SNAT_IN2OUT_NEXT_DROP;
 621           goto out;
 622         }
 623
 624       if (PREDICT_FALSE (value0.value == ~0ULL))
 625         {
 626           nat_ed_ses_key_t key;
 627           clib_bihash_kv_16_8_t s_kv, s_value;
 628
 629           key.as_u64[0] = 0;
 630           key.as_u64[1] = 0;
 631           if (icmp_get_ed_key (ip0, &key))
 632             {
 633               b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 634               next0 = SNAT_IN2OUT_NEXT_DROP;
 635               goto out;
 636             }
 637           key.fib_index = rx_fib_index0;
 638           s_kv.key[0] = key.as_u64[0];
 639           s_kv.key[1] = key.as_u64[1];
 640           if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 641             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 642                                     s_value.value);
 643           else
 644            {
 645               next0 = SNAT_IN2OUT_NEXT_DROP;
 646               goto out;
 647            }
 648         }
 649       else
 650         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 651                                 value0.value);
 652     }
 653
 654 out:
 655   *p_proto = key0.protocol;
 656   if (s0)
 657     *p_value = s0->out2in;
 658   *p_dont_translate = dont_translate;
 659   if (d)
 660     *(snat_session_t**)d = s0;
 661   return next0;
 662 }
 663
 664 /**
 665  * Get address and port values to be used for ICMP packet translation
 666  *
 667  * @param[in] sm                 NAT main
 668  * @param[in,out] node           NAT node runtime
 669  * @param[in] thread_index       thread index
 670  * @param[in,out] b0             buffer containing packet to be translated
 671  * @param[out] p_proto           protocol used for matching
 672  * @param[out] p_value           address and port after NAT translation
 673  * @param[out] p_dont_translate  if packet should not be translated
 674  * @param d                      optional parameter
 675  * @param e                      optional parameter
 676  */
 677 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 678                            u32 thread_index, vlib_buffer_t *b0,
 679                            ip4_header_t *ip0, u8 *p_proto,
 680                            snat_session_key_t *p_value,
 681                            u8 *p_dont_translate, void *d, void *e)
 682 {
 683   icmp46_header_t *icmp0;
 684   u32 sw_if_index0;
 685   u32 rx_fib_index0;
 686   snat_session_key_t key0;
 687   snat_session_key_t sm0;
 688   u8 dont_translate = 0;
 689   u8 is_addr_only;
 690   u32 next0 = ~0;
 691   int err;
 692
 693   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 694   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 695   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 696
 697   err = icmp_get_key (ip0, &key0);
 698   if (err != -1)
 699     {
 700       b0->error = node->errors[err];
 701       next0 = SNAT_IN2OUT_NEXT_DROP;
 702       goto out2;
 703     }
 704   key0.fib_index = rx_fib_index0;
 705
 706   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
 707     {
 708       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 709           IP_PROTOCOL_ICMP, rx_fib_index0)))
 710         {
 711           dont_translate = 1;
 712           goto out;
 713         }
 714
 715       if (icmp_is_error_message (icmp0))
 716         {
 717           next0 = SNAT_IN2OUT_NEXT_DROP;
 718           goto out;
 719         }
 720
 721       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 722       next0 = SNAT_IN2OUT_NEXT_DROP;
 723       goto out;
 724     }
 725
 726   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 727                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 728                     !icmp_is_error_message (icmp0)))
 729     {
 730       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 731       next0 = SNAT_IN2OUT_NEXT_DROP;
 732       goto out;
 733     }
 734
 735 out:
 736   *p_value = sm0;
 737 out2:
 738   *p_proto = key0.protocol;
 739   *p_dont_translate = dont_translate;
 740   return next0;
 741 }
 742
 743 static inline u32 icmp_in2out (snat_main_t *sm,
 744                                vlib_buffer_t * b0,
 745                                ip4_header_t * ip0,
 746                                icmp46_header_t * icmp0,
 747                                u32 sw_if_index0,
 748                                u32 rx_fib_index0,
 749                                vlib_node_runtime_t * node,
 750                                u32 next0,
 751                                u32 thread_index,
 752                                void *d,
 753                                void *e)
 754 {
 755   snat_session_key_t sm0;
 756   u8 protocol;
 757   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 758   ip4_header_t *inner_ip0;
 759   void *l4_header = 0;
 760   icmp46_header_t *inner_icmp0;
 761   u8 dont_translate;
 762   u32 new_addr0, old_addr0;
 763   u16 old_id0, new_id0;
 764   ip_csum_t sum0;
 765   u16 checksum0;
 766   u32 next0_tmp;
 767
 768   echo0 = (icmp_echo_header_t *)(icmp0+1);
 769
 770   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 771                                        &protocol, &sm0, &dont_translate, d, e);
 772   if (next0_tmp != ~0)
 773     next0 = next0_tmp;
 774   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 775     goto out;
 776
 777   sum0 = ip_incremental_checksum (0, icmp0,
 778                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 779   checksum0 = ~ip_csum_fold (sum0);
 780   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 781     {
 782       next0 = SNAT_IN2OUT_NEXT_DROP;
 783       goto out;
 784     }
 785
 786   old_addr0 = ip0->src_address.as_u32;
 787   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 788   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 789     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 790
 791   sum0 = ip0->checksum;
 792   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 793                          src_address /* changed member */);
 794   ip0->checksum = ip_csum_fold (sum0);
 795
 796   if (icmp0->checksum == 0)
 797     icmp0->checksum = 0xffff;
 798
 799   if (!icmp_is_error_message (icmp0))
 800     {
 801       new_id0 = sm0.port;
 802       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 803         {
 804           old_id0 = echo0->identifier;
 805           new_id0 = sm0.port;
 806           echo0->identifier = new_id0;
 807
 808           sum0 = icmp0->checksum;
 809           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 810                                  identifier);
 811           icmp0->checksum = ip_csum_fold (sum0);
 812         }
 813     }
 814   else
 815     {
 816       inner_ip0 = (ip4_header_t *)(echo0+1);
 817       l4_header = ip4_next_header (inner_ip0);
 818
 819       if (!ip4_header_checksum_is_valid (inner_ip0))
 820         {
 821           next0 = SNAT_IN2OUT_NEXT_DROP;
 822           goto out;
 823         }
 824
 825       old_addr0 = inner_ip0->dst_address.as_u32;
 826       inner_ip0->dst_address = sm0.addr;
 827       new_addr0 = inner_ip0->dst_address.as_u32;
 828
 829       sum0 = icmp0->checksum;
 830       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 831                              dst_address /* changed member */);
 832       icmp0->checksum = ip_csum_fold (sum0);
 833
 834       switch (protocol)
 835         {
 836           case SNAT_PROTOCOL_ICMP:
 837             inner_icmp0 = (icmp46_header_t*)l4_header;
 838             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 839
 840             old_id0 = inner_echo0->identifier;
 841             new_id0 = sm0.port;
 842             inner_echo0->identifier = new_id0;
 843
 844             sum0 = icmp0->checksum;
 845             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 846                                    identifier);
 847             icmp0->checksum = ip_csum_fold (sum0);
 848             break;
 849           case SNAT_PROTOCOL_UDP:
 850           case SNAT_PROTOCOL_TCP:
 851             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 852             new_id0 = sm0.port;
 853             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 854
 855             sum0 = icmp0->checksum;
 856             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 857                                    dst_port);
 858             icmp0->checksum = ip_csum_fold (sum0);
 859             break;
 860           default:
 861             ASSERT(0);
 862         }
 863     }
 864
 865 out:
 866   return next0;
 867 }
 868
 869 /**
 870  * @brief Hairpinning
 871  *
 872  * Hairpinning allows two endpoints on the internal side of the NAT to
 873  * communicate even if they only use each other's external IP addresses
 874  * and ports.
 875  *
 876  * @param sm     NAT main.
 877  * @param b0     Vlib buffer.
 878  * @param ip0    IP header.
 879  * @param udp0   UDP header.
 880  * @param tcp0   TCP header.
 881  * @param proto0 NAT protocol.
 882  */
 883 static inline int
 884 snat_hairpinning (snat_main_t *sm,
 885                   vlib_buffer_t * b0,
 886                   ip4_header_t * ip0,
 887                   udp_header_t * udp0,
 888                   tcp_header_t * tcp0,
 889                   u32 proto0)
 890 {
 891   snat_session_key_t key0, sm0;
 892   snat_session_t * s0;
 893   clib_bihash_kv_8_8_t kv0, value0;
 894   ip_csum_t sum0;
 895   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 896   u16 new_dst_port0, old_dst_port0;
 897
 898   key0.addr = ip0->dst_address;
 899   key0.port = udp0->dst_port;
 900   key0.protocol = proto0;
 901   key0.fib_index = sm->outside_fib_index;
 902   kv0.key = key0.as_u64;
 903
 904   /* Check if destination is static mappings */
 905   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 906     {
 907       new_dst_addr0 = sm0.addr.as_u32;
 908       new_dst_port0 = sm0.port;
 909       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 910     }
 911   /* or active session */
 912   else
 913     {
 914       if (sm->num_workers > 1)
 915         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 916       else
 917         ti = sm->num_workers;
 918
 919       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 920         {
 921           si = value0.value;
 922
 923           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 924           new_dst_addr0 = s0->in2out.addr.as_u32;
 925           new_dst_port0 = s0->in2out.port;
 926           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 927         }
 928     }
 929
 930   /* Destination is behind the same NAT, use internal address and port */
 931   if (new_dst_addr0)
 932     {
 933       old_dst_addr0 = ip0->dst_address.as_u32;
 934       ip0->dst_address.as_u32 = new_dst_addr0;
 935       sum0 = ip0->checksum;
 936       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 937                              ip4_header_t, dst_address);
 938       ip0->checksum = ip_csum_fold (sum0);
 939
 940       old_dst_port0 = tcp0->dst;
 941       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 942         {
 943           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 944             {
 945               tcp0->dst = new_dst_port0;
 946               sum0 = tcp0->checksum;
 947               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 948                                      ip4_header_t, dst_address);
 949               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 950                                      ip4_header_t /* cheat */, length);
 951               tcp0->checksum = ip_csum_fold(sum0);
 952             }
 953           else
 954             {
 955               udp0->dst_port = new_dst_port0;
 956               udp0->checksum = 0;
 957             }
 958         }
 959       else
 960         {
 961           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 962             {
 963               sum0 = tcp0->checksum;
 964               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 965                                      ip4_header_t, dst_address);
 966               tcp0->checksum = ip_csum_fold(sum0);
 967             }
 968         }
 969       return 1;
 970     }
 971   return 0;
 972 }
 973
 974 static inline void
 975 snat_icmp_hairpinning (snat_main_t *sm,
 976                        vlib_buffer_t * b0,
 977                        ip4_header_t * ip0,
 978                        icmp46_header_t * icmp0)
 979 {
 980   snat_session_key_t key0, sm0;
 981   clib_bihash_kv_8_8_t kv0, value0;
 982   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 983   ip_csum_t sum0;
 984   snat_session_t *s0;
 985
 986   if (!icmp_is_error_message (icmp0))
 987     {
 988       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 989       u16 icmp_id0 = echo0->identifier;
 990       key0.addr = ip0->dst_address;
 991       key0.port = icmp_id0;
 992       key0.protocol = SNAT_PROTOCOL_ICMP;
 993       key0.fib_index = sm->outside_fib_index;
 994       kv0.key = key0.as_u64;
 995
 996       if (sm->num_workers > 1)
 997         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 998       else
 999         ti = sm->num_workers;
1000
1001       /* Check if destination is in active sessions */
1002       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1003                                   &value0))
1004         {
1005           /* or static mappings */
1006           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1007             {
1008               new_dst_addr0 = sm0.addr.as_u32;
1009               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1010             }
1011         }
1012       else
1013         {
1014           si = value0.value;
1015
1016           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1017           new_dst_addr0 = s0->in2out.addr.as_u32;
1018           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1019           echo0->identifier = s0->in2out.port;
1020           sum0 = icmp0->checksum;
1021           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1022                                  icmp_echo_header_t, identifier);
1023           icmp0->checksum = ip_csum_fold (sum0);
1024         }
1025
1026       /* Destination is behind the same NAT, use internal address and port */
1027       if (new_dst_addr0)
1028         {
1029           old_dst_addr0 = ip0->dst_address.as_u32;
1030           ip0->dst_address.as_u32 = new_dst_addr0;
1031           sum0 = ip0->checksum;
1032           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1033                                  ip4_header_t, dst_address);
1034           ip0->checksum = ip_csum_fold (sum0);
1035         }
1036     }
1037
1038 }
1039
1040 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1041                                          vlib_buffer_t * b0,
1042                                          ip4_header_t * ip0,
1043                                          icmp46_header_t * icmp0,
1044                                          u32 sw_if_index0,
1045                                          u32 rx_fib_index0,
1046                                          vlib_node_runtime_t * node,
1047                                          u32 next0,
1048                                          f64 now,
1049                                          u32 thread_index,
1050                                          snat_session_t ** p_s0)
1051 {
1052   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1053                       next0, thread_index, p_s0, 0);
1054   snat_session_t * s0 = *p_s0;
1055   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1056     {
1057       /* Hairpinning */
1058       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1059         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1060       /* Accounting */
1061       s0->last_heard = now;
1062       s0->total_pkts++;
1063       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1064       /* Per-user LRU list maintenance */
1065       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1066                          s0->per_user_index);
1067       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1068                           s0->per_user_list_head_index,
1069                           s0->per_user_index);
1070     }
1071   return next0;
1072 }
1073 static inline void
1074 snat_hairpinning_unknown_proto (snat_main_t *sm,
1075                                 vlib_buffer_t * b,
1076                                 ip4_header_t * ip)
1077 {
1078   u32 old_addr, new_addr = 0, ti = 0;
1079   clib_bihash_kv_8_8_t kv, value;
1080   clib_bihash_kv_16_8_t s_kv, s_value;
1081   nat_ed_ses_key_t key;
1082   snat_session_key_t m_key;
1083   snat_static_mapping_t *m;
1084   ip_csum_t sum;
1085   snat_session_t *s;
1086
1087   old_addr = ip->dst_address.as_u32;
1088   key.l_addr.as_u32 = ip->dst_address.as_u32;
1089   key.r_addr.as_u32 = ip->src_address.as_u32;
1090   key.fib_index = sm->outside_fib_index;
1091   key.proto = ip->protocol;
1092   key.r_port = 0;
1093   key.l_port = 0;
1094   s_kv.key[0] = key.as_u64[0];
1095   s_kv.key[1] = key.as_u64[1];
1096   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1097     {
1098       m_key.addr = ip->dst_address;
1099       m_key.fib_index = sm->outside_fib_index;
1100       m_key.port = 0;
1101       m_key.protocol = 0;
1102       kv.key = m_key.as_u64;
1103       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1104         return;
1105
1106       m = pool_elt_at_index (sm->static_mappings, value.value);
1107       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1108         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1109       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1110     }
1111   else
1112     {
1113       if (sm->num_workers > 1)
1114         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1115       else
1116         ti = sm->num_workers;
1117
1118       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1119       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1120         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1121       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1122     }
1123   sum = ip->checksum;
1124   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1125   ip->checksum = ip_csum_fold (sum);
1126 }
1127
1128 static snat_session_t *
1129 snat_in2out_unknown_proto (snat_main_t *sm,
1130                            vlib_buffer_t * b,
1131                            ip4_header_t * ip,
1132                            u32 rx_fib_index,
1133                            u32 thread_index,
1134                            f64 now,
1135                            vlib_main_t * vm,
1136                            vlib_node_runtime_t * node)
1137 {
1138   clib_bihash_kv_8_8_t kv, value;
1139   clib_bihash_kv_16_8_t s_kv, s_value;
1140   snat_static_mapping_t *m;
1141   snat_session_key_t m_key;
1142   u32 old_addr, new_addr = 0;
1143   ip_csum_t sum;
1144   snat_user_t *u;
1145   dlist_elt_t *head, *elt;
1146   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1147   u32 elt_index, head_index, ses_index;
1148   snat_session_t * s;
1149   nat_ed_ses_key_t key;
1150   u32 address_index = ~0;
1151   int i;
1152   u8 is_sm = 0;
1153
1154   old_addr = ip->src_address.as_u32;
1155
1156   key.l_addr = ip->src_address;
1157   key.r_addr = ip->dst_address;
1158   key.fib_index = rx_fib_index;
1159   key.proto = ip->protocol;
1160   key.l_port = 0;
1161   key.r_port = 0;
1162   s_kv.key[0] = key.as_u64[0];
1163   s_kv.key[1] = key.as_u64[1];
1164
1165   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1166     {
1167       s = pool_elt_at_index (tsm->sessions, s_value.value);
1168       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1169     }
1170   else
1171     {
1172       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1173         {
1174           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1175           nat_ipfix_logging_max_sessions(sm->max_translations);
1176           return 0;
1177         }
1178
1179       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1180                                   thread_index);
1181       if (!u)
1182         {
1183           clib_warning ("create NAT user failed");
1184           return 0;
1185         }
1186
1187       m_key.addr = ip->src_address;
1188       m_key.port = 0;
1189       m_key.protocol = 0;
1190       m_key.fib_index = rx_fib_index;
1191       kv.key = m_key.as_u64;
1192
1193       /* Try to find static mapping first */
1194       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1195         {
1196           m = pool_elt_at_index (sm->static_mappings, value.value);
1197           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1198           is_sm = 1;
1199           goto create_ses;
1200         }
1201       /* Fallback to 3-tuple key */
1202       else
1203         {
1204           /* Choose same out address as for TCP/UDP session to same destination */
1205           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1206             {
1207               head_index = u->sessions_per_user_list_head_index;
1208               head = pool_elt_at_index (tsm->list_pool, head_index);
1209               elt_index = head->next;
1210               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1211               ses_index = elt->value;
1212               while (ses_index != ~0)
1213                 {
1214                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1215                   elt_index = elt->next;
1216                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1217                   ses_index = elt->value;
1218
1219                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1220                     {
1221                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1222                       address_index = s->outside_address_index;
1223
1224                       key.fib_index = sm->outside_fib_index;
1225                       key.l_addr.as_u32 = new_addr;
1226                       s_kv.key[0] = key.as_u64[0];
1227                       s_kv.key[1] = key.as_u64[1];
1228                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1229                         break;
1230
1231                       goto create_ses;
1232                     }
1233                 }
1234             }
1235           key.fib_index = sm->outside_fib_index;
1236           for (i = 0; i < vec_len (sm->addresses); i++)
1237             {
1238               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1239               s_kv.key[0] = key.as_u64[0];
1240               s_kv.key[1] = key.as_u64[1];
1241               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1242                 {
1243                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1244                   address_index = i;
1245                   goto create_ses;
1246                 }
1247             }
1248           return 0;
1249         }
1250
1251 create_ses:
1252       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1253       if (!s)
1254         {
1255           clib_warning ("create NAT session failed");
1256           return 0;
1257         }
1258
1259       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1260       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1261       s->outside_address_index = address_index;
1262       s->out2in.addr.as_u32 = new_addr;
1263       s->out2in.fib_index = sm->outside_fib_index;
1264       s->in2out.addr.as_u32 = old_addr;
1265       s->in2out.fib_index = rx_fib_index;
1266       s->in2out.port = s->out2in.port = ip->protocol;
1267       if (is_sm)
1268         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1269       user_session_increment (sm, u, is_sm);
1270
1271       /* Add to lookup tables */
1272       key.l_addr.as_u32 = old_addr;
1273       key.r_addr = ip->dst_address;
1274       key.proto = ip->protocol;
1275       key.fib_index = rx_fib_index;
1276       s_kv.key[0] = key.as_u64[0];
1277       s_kv.key[1] = key.as_u64[1];
1278       s_kv.value = s - tsm->sessions;
1279       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1280         clib_warning ("in2out key add failed");
1281
1282       key.l_addr.as_u32 = new_addr;
1283       key.fib_index = sm->outside_fib_index;
1284       s_kv.key[0] = key.as_u64[0];
1285       s_kv.key[1] = key.as_u64[1];
1286       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1287         clib_warning ("out2in key add failed");
1288   }
1289
1290   /* Update IP checksum */
1291   sum = ip->checksum;
1292   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1293   ip->checksum = ip_csum_fold (sum);
1294
1295   /* Accounting */
1296   s->last_heard = now;
1297   s->total_pkts++;
1298   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1299   /* Per-user LRU list maintenance */
1300   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1301   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1302                       s->per_user_index);
1303
1304   /* Hairpinning */
1305   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1306     snat_hairpinning_unknown_proto(sm, b, ip);
1307
1308   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1309     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1310
1311   return s;
1312 }
1313
1314 static snat_session_t *
1315 snat_in2out_lb (snat_main_t *sm,
1316                 vlib_buffer_t * b,
1317                 ip4_header_t * ip,
1318                 u32 rx_fib_index,
1319                 u32 thread_index,
1320                 f64 now,
1321                 vlib_main_t * vm,
1322                 vlib_node_runtime_t * node)
1323 {
1324   nat_ed_ses_key_t key;
1325   clib_bihash_kv_16_8_t s_kv, s_value;
1326   udp_header_t *udp = ip4_next_header (ip);
1327   tcp_header_t *tcp = (tcp_header_t *) udp;
1328   snat_session_t *s = 0;
1329   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1330   u32 old_addr, new_addr;
1331   u16 new_port, old_port;
1332   ip_csum_t sum;
1333   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1334   snat_session_key_t e_key, l_key;
1335   snat_user_t *u;
1336   u8 lb;
1337
1338   old_addr = ip->src_address.as_u32;
1339
1340   key.l_addr = ip->src_address;
1341   key.r_addr = ip->dst_address;
1342   key.fib_index = rx_fib_index;
1343   key.proto = ip->protocol;
1344   key.r_port = udp->dst_port;
1345   key.l_port = udp->src_port;
1346   s_kv.key[0] = key.as_u64[0];
1347   s_kv.key[1] = key.as_u64[1];
1348
1349   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1350     {
1351       if (s_value.value == ~0ULL)
1352         return 0;
1353       s = pool_elt_at_index (tsm->sessions, s_value.value);
1354     }
1355   else
1356     {
1357       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1358         {
1359           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1360           nat_ipfix_logging_max_sessions(sm->max_translations);
1361           return 0;
1362         }
1363
1364       l_key.addr = ip->src_address;
1365       l_key.port = udp->src_port;
1366       l_key.protocol = proto;
1367       l_key.fib_index = rx_fib_index;
1368       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0, &lb))
1369         return 0;
1370
1371       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1372                                   thread_index);
1373       if (!u)
1374         {
1375           clib_warning ("create NAT user failed");
1376           return 0;
1377         }
1378
1379       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1380       if (!s)
1381         {
1382           clib_warning ("create NAT session failed");
1383           return 0;
1384         }
1385
1386       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1387       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1388       if (lb)
1389         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1390       s->outside_address_index = ~0;
1391       s->in2out = l_key;
1392       s->out2in = e_key;
1393       s->out2in.protocol = l_key.protocol;
1394       user_session_increment (sm, u, 1 /* static */);
1395
1396       /* Add to lookup tables */
1397       s_kv.value = s - tsm->sessions;
1398       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1399         clib_warning ("in2out-ed key add failed");
1400
1401       key.l_addr = e_key.addr;
1402       key.fib_index = e_key.fib_index;
1403       key.l_port = e_key.port;
1404       s_kv.key[0] = key.as_u64[0];
1405       s_kv.key[1] = key.as_u64[1];
1406       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1407         clib_warning ("out2in-ed key add failed");
1408     }
1409
1410   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1411
1412   /* Update IP checksum */
1413   sum = ip->checksum;
1414   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1415   if (is_twice_nat_session (s))
1416     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1417                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1418   ip->checksum = ip_csum_fold (sum);
1419
1420   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1421     {
1422       old_port = tcp->src_port;
1423       tcp->src_port = s->out2in.port;
1424       new_port = tcp->src_port;
1425
1426       sum = tcp->checksum;
1427       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1428       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1429       if (is_twice_nat_session (s))
1430         {
1431           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1432                                 s->ext_host_addr.as_u32, ip4_header_t,
1433                                 dst_address);
1434           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1435                                 ip4_header_t, length);
1436           tcp->dst_port = s->ext_host_port;
1437           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1438         }
1439       tcp->checksum = ip_csum_fold(sum);
1440     }
1441   else
1442     {
1443       udp->src_port = s->out2in.port;
1444       if (is_twice_nat_session (s))
1445         {
1446           udp->dst_port = s->ext_host_port;
1447           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1448         }
1449       udp->checksum = 0;
1450     }
1451
1452   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1453     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1454
1455   /* Accounting */
1456   s->last_heard = now;
1457   s->total_pkts++;
1458   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1459   /* Per-user LRU list maintenance */
1460   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1461   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1462                       s->per_user_index);
1463   return s;
1464 }
1465
1466 static inline uword
1467 snat_in2out_node_fn_inline (vlib_main_t * vm,
1468                             vlib_node_runtime_t * node,
1469                             vlib_frame_t * frame, int is_slow_path,
1470                             int is_output_feature)
1471 {
1472   u32 n_left_from, * from, * to_next;
1473   snat_in2out_next_t next_index;
1474   u32 pkts_processed = 0;
1475   snat_main_t * sm = &snat_main;
1476   f64 now = vlib_time_now (vm);
1477   u32 stats_node_index;
1478   u32 thread_index = vlib_get_thread_index ();
1479
1480   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1481     snat_in2out_node.index;
1482
1483   from = vlib_frame_vector_args (frame);
1484   n_left_from = frame->n_vectors;
1485   next_index = node->cached_next_index;
1486
1487   while (n_left_from > 0)
1488     {
1489       u32 n_left_to_next;
1490
1491       vlib_get_next_frame (vm, node, next_index,
1492                            to_next, n_left_to_next);
1493
1494       while (n_left_from >= 4 && n_left_to_next >= 2)
1495         {
1496           u32 bi0, bi1;
1497           vlib_buffer_t * b0, * b1;
1498           u32 next0, next1;
1499           u32 sw_if_index0, sw_if_index1;
1500           ip4_header_t * ip0, * ip1;
1501           ip_csum_t sum0, sum1;
1502           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1503           u16 old_port0, new_port0, old_port1, new_port1;
1504           udp_header_t * udp0, * udp1;
1505           tcp_header_t * tcp0, * tcp1;
1506           icmp46_header_t * icmp0, * icmp1;
1507           snat_session_key_t key0, key1;
1508           u32 rx_fib_index0, rx_fib_index1;
1509           u32 proto0, proto1;
1510           snat_session_t * s0 = 0, * s1 = 0;
1511           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1512           u32 iph_offset0 = 0, iph_offset1 = 0;
1513
1514           /* Prefetch next iteration. */
1515           {
1516             vlib_buffer_t * p2, * p3;
1517
1518             p2 = vlib_get_buffer (vm, from[2]);
1519             p3 = vlib_get_buffer (vm, from[3]);
1520
1521             vlib_prefetch_buffer_header (p2, LOAD);
1522             vlib_prefetch_buffer_header (p3, LOAD);
1523
1524             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1525             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1526           }
1527
1528           /* speculatively enqueue b0 and b1 to the current next frame */
1529           to_next[0] = bi0 = from[0];
1530           to_next[1] = bi1 = from[1];
1531           from += 2;
1532           to_next += 2;
1533           n_left_from -= 2;
1534           n_left_to_next -= 2;
1535
1536           b0 = vlib_get_buffer (vm, bi0);
1537           b1 = vlib_get_buffer (vm, bi1);
1538
1539           if (is_output_feature)
1540             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1541
1542           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1543                  iph_offset0);
1544
1545           udp0 = ip4_next_header (ip0);
1546           tcp0 = (tcp_header_t *) udp0;
1547           icmp0 = (icmp46_header_t *) udp0;
1548
1549           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1550           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1551                                    sw_if_index0);
1552
1553           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1554
1555           if (PREDICT_FALSE(ip0->ttl == 1))
1556             {
1557               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1558               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1559                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1560                                            0);
1561               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1562               goto trace00;
1563             }
1564
1565           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1566
1567           /* Next configured feature, probably ip4-lookup */
1568           if (is_slow_path)
1569             {
1570               if (PREDICT_FALSE (proto0 == ~0))
1571                 {
1572                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1573                                                   thread_index, now, vm, node);
1574                   if (!s0)
1575                     next0 = SNAT_IN2OUT_NEXT_DROP;
1576                   goto trace00;
1577                 }
1578
1579               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1580                 {
1581                   next0 = icmp_in2out_slow_path
1582                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1583                      node, next0, now, thread_index, &s0);
1584                   goto trace00;
1585                 }
1586             }
1587           else
1588             {
1589               if (is_output_feature)
1590                 {
1591                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0)))
1592                     goto trace00;
1593                 }
1594
1595               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1596                 {
1597                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1598                   goto trace00;
1599                 }
1600
1601               if (ip4_is_fragment (ip0))
1602                 {
1603                   next0 = SNAT_IN2OUT_NEXT_REASS;
1604                   goto trace00;
1605                 }
1606             }
1607
1608           key0.addr = ip0->src_address;
1609           key0.port = udp0->src_port;
1610           key0.protocol = proto0;
1611           key0.fib_index = rx_fib_index0;
1612
1613           kv0.key = key0.as_u64;
1614
1615           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1616               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1617             {
1618               if (is_slow_path)
1619                 {
1620                   if (is_output_feature)
1621                     {
1622                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1623                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1624                         goto trace00;
1625                     }
1626                   else
1627                     {
1628                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1629                           ip0, proto0, rx_fib_index0, thread_index)))
1630                         goto trace00;
1631                     }
1632
1633                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1634                                      &s0, node, next0, thread_index);
1635                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1636                     goto trace00;
1637                 }
1638               else
1639                 {
1640                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1641                   goto trace00;
1642                 }
1643             }
1644           else
1645             {
1646               if (PREDICT_FALSE (value0.value == ~0ULL))
1647                 {
1648                   if (is_slow_path)
1649                     {
1650                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1651                                           thread_index, now, vm, node);
1652                       if (!s0 && !sm->forwarding_enabled)
1653                         next0 = SNAT_IN2OUT_NEXT_DROP;
1654                       goto trace00;
1655                     }
1656                   else
1657                     {
1658                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1659                       goto trace00;
1660                     }
1661                 }
1662               else
1663                 {
1664                   s0 = pool_elt_at_index (
1665                     sm->per_thread_data[thread_index].sessions,
1666                     value0.value);
1667                 }
1668             }
1669
1670           b0->flags |= VNET_BUFFER_F_IS_NATED;
1671
1672           old_addr0 = ip0->src_address.as_u32;
1673           ip0->src_address = s0->out2in.addr;
1674           new_addr0 = ip0->src_address.as_u32;
1675           if (!is_output_feature)
1676             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1677
1678           sum0 = ip0->checksum;
1679           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1680                                  ip4_header_t,
1681                                  src_address /* changed member */);
1682           ip0->checksum = ip_csum_fold (sum0);
1683
1684           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1685             {
1686               old_port0 = tcp0->src_port;
1687               tcp0->src_port = s0->out2in.port;
1688               new_port0 = tcp0->src_port;
1689
1690               sum0 = tcp0->checksum;
1691               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1692                                      ip4_header_t,
1693                                      dst_address /* changed member */);
1694               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1695                                      ip4_header_t /* cheat */,
1696                                      length /* changed member */);
1697               tcp0->checksum = ip_csum_fold(sum0);
1698             }
1699           else
1700             {
1701               old_port0 = udp0->src_port;
1702               udp0->src_port = s0->out2in.port;
1703               udp0->checksum = 0;
1704             }
1705
1706           /* Accounting */
1707           s0->last_heard = now;
1708           s0->total_pkts++;
1709           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1710           /* Per-user LRU list maintenance */
1711           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1712                              s0->per_user_index);
1713           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1714                               s0->per_user_list_head_index,
1715                               s0->per_user_index);
1716         trace00:
1717
1718           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1719                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1720             {
1721               snat_in2out_trace_t *t =
1722                  vlib_add_trace (vm, node, b0, sizeof (*t));
1723               t->is_slow_path = is_slow_path;
1724               t->sw_if_index = sw_if_index0;
1725               t->next_index = next0;
1726                   t->session_index = ~0;
1727               if (s0)
1728                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1729             }
1730
1731           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1732
1733           if (is_output_feature)
1734             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1735
1736           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1737                  iph_offset1);
1738
1739           udp1 = ip4_next_header (ip1);
1740           tcp1 = (tcp_header_t *) udp1;
1741           icmp1 = (icmp46_header_t *) udp1;
1742
1743           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1744           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1745                                    sw_if_index1);
1746
1747           if (PREDICT_FALSE(ip1->ttl == 1))
1748             {
1749               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1750               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1751                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1752                                            0);
1753               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1754               goto trace01;
1755             }
1756
1757           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1758
1759           /* Next configured feature, probably ip4-lookup */
1760           if (is_slow_path)
1761             {
1762               if (PREDICT_FALSE (proto1 == ~0))
1763                 {
1764                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1765                                                   thread_index, now, vm, node);
1766                   if (!s1)
1767                     next1 = SNAT_IN2OUT_NEXT_DROP;
1768                   goto trace01;
1769                 }
1770
1771               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1772                 {
1773                   next1 = icmp_in2out_slow_path
1774                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1775                      next1, now, thread_index, &s1);
1776                   goto trace01;
1777                 }
1778             }
1779           else
1780             {
1781               if (is_output_feature)
1782                 {
1783                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip1)))
1784                     goto trace01;
1785                 }
1786
1787               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1788                 {
1789                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1790                   goto trace01;
1791                 }
1792
1793               if (ip4_is_fragment (ip1))
1794                 {
1795                   next1 = SNAT_IN2OUT_NEXT_REASS;
1796                   goto trace01;
1797                 }
1798             }
1799
1800           key1.addr = ip1->src_address;
1801           key1.port = udp1->src_port;
1802           key1.protocol = proto1;
1803           key1.fib_index = rx_fib_index1;
1804
1805           kv1.key = key1.as_u64;
1806
1807             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1808                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1809             {
1810               if (is_slow_path)
1811                 {
1812                   if (is_output_feature)
1813                     {
1814                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1815                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1816                         goto trace01;
1817                     }
1818                   else
1819                     {
1820                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1821                           ip1, proto1, rx_fib_index1, thread_index)))
1822                         goto trace01;
1823                     }
1824
1825                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1826                                      &s1, node, next1, thread_index);
1827                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1828                     goto trace01;
1829                 }
1830               else
1831                 {
1832                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1833                   goto trace01;
1834                 }
1835             }
1836           else
1837             {
1838               if (PREDICT_FALSE (value1.value == ~0ULL))
1839                 {
1840                   if (is_slow_path)
1841                     {
1842                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1843                                           thread_index, now, vm, node);
1844                       if (!s1 && !sm->forwarding_enabled)
1845                         next1 = SNAT_IN2OUT_NEXT_DROP;
1846                       goto trace01;
1847                     }
1848                   else
1849                     {
1850                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1851                       goto trace01;
1852                     }
1853                 }
1854               else
1855                 {
1856                   s1 = pool_elt_at_index (
1857                     sm->per_thread_data[thread_index].sessions,
1858                     value1.value);
1859                 }
1860             }
1861
1862           b1->flags |= VNET_BUFFER_F_IS_NATED;
1863
1864           old_addr1 = ip1->src_address.as_u32;
1865           ip1->src_address = s1->out2in.addr;
1866           new_addr1 = ip1->src_address.as_u32;
1867           if (!is_output_feature)
1868             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1869
1870           sum1 = ip1->checksum;
1871           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1872                                  ip4_header_t,
1873                                  src_address /* changed member */);
1874           ip1->checksum = ip_csum_fold (sum1);
1875
1876           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1877             {
1878               old_port1 = tcp1->src_port;
1879               tcp1->src_port = s1->out2in.port;
1880               new_port1 = tcp1->src_port;
1881
1882               sum1 = tcp1->checksum;
1883               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1884                                      ip4_header_t,
1885                                      dst_address /* changed member */);
1886               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1887                                      ip4_header_t /* cheat */,
1888                                      length /* changed member */);
1889               tcp1->checksum = ip_csum_fold(sum1);
1890             }
1891           else
1892             {
1893               old_port1 = udp1->src_port;
1894               udp1->src_port = s1->out2in.port;
1895               udp1->checksum = 0;
1896             }
1897
1898           /* Accounting */
1899           s1->last_heard = now;
1900           s1->total_pkts++;
1901           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1902           /* Per-user LRU list maintenance */
1903           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1904                              s1->per_user_index);
1905           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1906                               s1->per_user_list_head_index,
1907                               s1->per_user_index);
1908         trace01:
1909
1910           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1911                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1912             {
1913               snat_in2out_trace_t *t =
1914                  vlib_add_trace (vm, node, b1, sizeof (*t));
1915               t->sw_if_index = sw_if_index1;
1916               t->next_index = next1;
1917               t->session_index = ~0;
1918               if (s1)
1919                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1920             }
1921
1922           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1923
1924           /* verify speculative enqueues, maybe switch current next frame */
1925           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1926                                            to_next, n_left_to_next,
1927                                            bi0, bi1, next0, next1);
1928         }
1929
1930       while (n_left_from > 0 && n_left_to_next > 0)
1931         {
1932           u32 bi0;
1933           vlib_buffer_t * b0;
1934           u32 next0;
1935           u32 sw_if_index0;
1936           ip4_header_t * ip0;
1937           ip_csum_t sum0;
1938           u32 new_addr0, old_addr0;
1939           u16 old_port0, new_port0;
1940           udp_header_t * udp0;
1941           tcp_header_t * tcp0;
1942           icmp46_header_t * icmp0;
1943           snat_session_key_t key0;
1944           u32 rx_fib_index0;
1945           u32 proto0;
1946           snat_session_t * s0 = 0;
1947           clib_bihash_kv_8_8_t kv0, value0;
1948           u32 iph_offset0 = 0;
1949
1950           /* speculatively enqueue b0 to the current next frame */
1951           bi0 = from[0];
1952           to_next[0] = bi0;
1953           from += 1;
1954           to_next += 1;
1955           n_left_from -= 1;
1956           n_left_to_next -= 1;
1957
1958           b0 = vlib_get_buffer (vm, bi0);
1959           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1960
1961           if (is_output_feature)
1962             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1963
1964           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1965                  iph_offset0);
1966
1967           udp0 = ip4_next_header (ip0);
1968           tcp0 = (tcp_header_t *) udp0;
1969           icmp0 = (icmp46_header_t *) udp0;
1970
1971           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1972           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1973                                    sw_if_index0);
1974
1975           if (PREDICT_FALSE(ip0->ttl == 1))
1976             {
1977               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1978               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1979                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1980                                            0);
1981               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1982               goto trace0;
1983             }
1984
1985           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1986
1987           /* Next configured feature, probably ip4-lookup */
1988           if (is_slow_path)
1989             {
1990               if (PREDICT_FALSE (proto0 == ~0))
1991                 {
1992                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1993                                                   thread_index, now, vm, node);
1994                   if (!s0)
1995                     next0 = SNAT_IN2OUT_NEXT_DROP;
1996                   goto trace0;
1997                 }
1998
1999               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2000                 {
2001                   next0 = icmp_in2out_slow_path
2002                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2003                      next0, now, thread_index, &s0);
2004                   goto trace0;
2005                 }
2006             }
2007           else
2008             {
2009                if (is_output_feature)
2010                 {
2011                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0)))
2012                     goto trace0;
2013                 }
2014
2015               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2016                 {
2017                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2018                   goto trace0;
2019                 }
2020
2021               if (ip4_is_fragment (ip0))
2022                 {
2023                   next0 = SNAT_IN2OUT_NEXT_REASS;
2024                   goto trace0;
2025                 }
2026             }
2027
2028           key0.addr = ip0->src_address;
2029           key0.port = udp0->src_port;
2030           key0.protocol = proto0;
2031           key0.fib_index = rx_fib_index0;
2032
2033           kv0.key = key0.as_u64;
2034
2035           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2036                                       &kv0, &value0))
2037             {
2038               if (is_slow_path)
2039                 {
2040                   if (is_output_feature)
2041                     {
2042                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
2043                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
2044                         goto trace0;
2045                     }
2046                   else
2047                     {
2048                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2049                           ip0, proto0, rx_fib_index0, thread_index)))
2050                         goto trace0;
2051                     }
2052
2053                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2054                                      &s0, node, next0, thread_index);
2055
2056                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2057                     goto trace0;
2058                 }
2059               else
2060                 {
2061                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2062                   goto trace0;
2063                 }
2064             }
2065           else
2066             {
2067               if (PREDICT_FALSE (value0.value == ~0ULL))
2068                 {
2069                   if (is_slow_path)
2070                     {
2071                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2072                                           thread_index, now, vm, node);
2073                       if (!s0 && !sm->forwarding_enabled)
2074                         next0 = SNAT_IN2OUT_NEXT_DROP;
2075                       goto trace0;
2076                     }
2077                   else
2078                     {
2079                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2080                       goto trace0;
2081                     }
2082                 }
2083               else
2084                 {
2085                   s0 = pool_elt_at_index (
2086                     sm->per_thread_data[thread_index].sessions,
2087                     value0.value);
2088                 }
2089             }
2090
2091           b0->flags |= VNET_BUFFER_F_IS_NATED;
2092
2093           old_addr0 = ip0->src_address.as_u32;
2094           ip0->src_address = s0->out2in.addr;
2095           new_addr0 = ip0->src_address.as_u32;
2096           if (!is_output_feature)
2097             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2098
2099           sum0 = ip0->checksum;
2100           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2101                                  ip4_header_t,
2102                                  src_address /* changed member */);
2103           ip0->checksum = ip_csum_fold (sum0);
2104
2105           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2106             {
2107               old_port0 = tcp0->src_port;
2108               tcp0->src_port = s0->out2in.port;
2109               new_port0 = tcp0->src_port;
2110
2111               sum0 = tcp0->checksum;
2112               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2113                                      ip4_header_t,
2114                                      dst_address /* changed member */);
2115               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2116                                      ip4_header_t /* cheat */,
2117                                      length /* changed member */);
2118               tcp0->checksum = ip_csum_fold(sum0);
2119             }
2120           else
2121             {
2122               old_port0 = udp0->src_port;
2123               udp0->src_port = s0->out2in.port;
2124               udp0->checksum = 0;
2125             }
2126
2127           /* Accounting */
2128           s0->last_heard = now;
2129           s0->total_pkts++;
2130           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2131           /* Per-user LRU list maintenance */
2132           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2133                              s0->per_user_index);
2134           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2135                               s0->per_user_list_head_index,
2136                               s0->per_user_index);
2137
2138         trace0:
2139           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2140                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2141             {
2142               snat_in2out_trace_t *t =
2143                  vlib_add_trace (vm, node, b0, sizeof (*t));
2144               t->is_slow_path = is_slow_path;
2145               t->sw_if_index = sw_if_index0;
2146               t->next_index = next0;
2147                   t->session_index = ~0;
2148               if (s0)
2149                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2150             }
2151
2152           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2153
2154           /* verify speculative enqueue, maybe switch current next frame */
2155           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2156                                            to_next, n_left_to_next,
2157                                            bi0, next0);
2158         }
2159
2160       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2161     }
2162
2163   vlib_node_increment_counter (vm, stats_node_index,
2164                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2165                                pkts_processed);
2166   return frame->n_vectors;
2167 }
2168
2169 static uword
2170 snat_in2out_fast_path_fn (vlib_main_t * vm,
2171                           vlib_node_runtime_t * node,
2172                           vlib_frame_t * frame)
2173 {
2174   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2175 }
2176
2177 VLIB_REGISTER_NODE (snat_in2out_node) = {
2178   .function = snat_in2out_fast_path_fn,
2179   .name = "nat44-in2out",
2180   .vector_size = sizeof (u32),
2181   .format_trace = format_snat_in2out_trace,
2182   .type = VLIB_NODE_TYPE_INTERNAL,
2183
2184   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2185   .error_strings = snat_in2out_error_strings,
2186
2187   .runtime_data_bytes = sizeof (snat_runtime_t),
2188
2189   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2190
2191   /* edit / add dispositions here */
2192   .next_nodes = {
2193     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2194     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2195     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2196     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2197     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2198   },
2199 };
2200
2201 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2202
2203 static uword
2204 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2205                                  vlib_node_runtime_t * node,
2206                                  vlib_frame_t * frame)
2207 {
2208   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2209 }
2210
2211 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2212   .function = snat_in2out_output_fast_path_fn,
2213   .name = "nat44-in2out-output",
2214   .vector_size = sizeof (u32),
2215   .format_trace = format_snat_in2out_trace,
2216   .type = VLIB_NODE_TYPE_INTERNAL,
2217
2218   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2219   .error_strings = snat_in2out_error_strings,
2220
2221   .runtime_data_bytes = sizeof (snat_runtime_t),
2222
2223   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2224
2225   /* edit / add dispositions here */
2226   .next_nodes = {
2227     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2228     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2229     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2230     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2231     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2232   },
2233 };
2234
2235 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2236                               snat_in2out_output_fast_path_fn);
2237
2238 static uword
2239 snat_in2out_slow_path_fn (vlib_main_t * vm,
2240                           vlib_node_runtime_t * node,
2241                           vlib_frame_t * frame)
2242 {
2243   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2244 }
2245
2246 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2247   .function = snat_in2out_slow_path_fn,
2248   .name = "nat44-in2out-slowpath",
2249   .vector_size = sizeof (u32),
2250   .format_trace = format_snat_in2out_trace,
2251   .type = VLIB_NODE_TYPE_INTERNAL,
2252
2253   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2254   .error_strings = snat_in2out_error_strings,
2255
2256   .runtime_data_bytes = sizeof (snat_runtime_t),
2257
2258   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2259
2260   /* edit / add dispositions here */
2261   .next_nodes = {
2262     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2263     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2264     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2265     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2266     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2267   },
2268 };
2269
2270 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2271                               snat_in2out_slow_path_fn);
2272
2273 static uword
2274 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2275                                  vlib_node_runtime_t * node,
2276                                  vlib_frame_t * frame)
2277 {
2278   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2279 }
2280
2281 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2282   .function = snat_in2out_output_slow_path_fn,
2283   .name = "nat44-in2out-output-slowpath",
2284   .vector_size = sizeof (u32),
2285   .format_trace = format_snat_in2out_trace,
2286   .type = VLIB_NODE_TYPE_INTERNAL,
2287
2288   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2289   .error_strings = snat_in2out_error_strings,
2290
2291   .runtime_data_bytes = sizeof (snat_runtime_t),
2292
2293   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2294
2295   /* edit / add dispositions here */
2296   .next_nodes = {
2297     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2298     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2299     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2300     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2301     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2302   },
2303 };
2304
2305 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2306                               snat_in2out_output_slow_path_fn);
2307
2308 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2309
2310 static uword
2311 nat44_hairpinning_fn (vlib_main_t * vm,
2312                       vlib_node_runtime_t * node,
2313                       vlib_frame_t * frame)
2314 {
2315   u32 n_left_from, * from, * to_next;
2316   snat_in2out_next_t next_index;
2317   u32 pkts_processed = 0;
2318   snat_main_t * sm = &snat_main;
2319   vnet_feature_main_t *fm = &feature_main;
2320   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2321   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2322
2323   from = vlib_frame_vector_args (frame);
2324   n_left_from = frame->n_vectors;
2325   next_index = node->cached_next_index;
2326
2327   while (n_left_from > 0)
2328     {
2329       u32 n_left_to_next;
2330
2331       vlib_get_next_frame (vm, node, next_index,
2332                            to_next, n_left_to_next);
2333
2334       while (n_left_from > 0 && n_left_to_next > 0)
2335         {
2336           u32 bi0;
2337           vlib_buffer_t * b0;
2338           u32 next0;
2339           ip4_header_t * ip0;
2340           u32 proto0;
2341           udp_header_t * udp0;
2342           tcp_header_t * tcp0;
2343
2344           /* speculatively enqueue b0 to the current next frame */
2345           bi0 = from[0];
2346           to_next[0] = bi0;
2347           from += 1;
2348           to_next += 1;
2349           n_left_from -= 1;
2350           n_left_to_next -= 1;
2351
2352           b0 = vlib_get_buffer (vm, bi0);
2353           ip0 = vlib_buffer_get_current (b0);
2354           udp0 = ip4_next_header (ip0);
2355           tcp0 = (tcp_header_t *) udp0;
2356
2357           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2358
2359           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2360                                 &next0, 0);
2361
2362           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2363             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2364
2365           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2366
2367           /* verify speculative enqueue, maybe switch current next frame */
2368           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2369                                            to_next, n_left_to_next,
2370                                            bi0, next0);
2371          }
2372
2373       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2374     }
2375
2376   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2377                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2378                                pkts_processed);
2379   return frame->n_vectors;
2380 }
2381
2382 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2383   .function = nat44_hairpinning_fn,
2384   .name = "nat44-hairpinning",
2385   .vector_size = sizeof (u32),
2386   .type = VLIB_NODE_TYPE_INTERNAL,
2387   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2388   .error_strings = snat_in2out_error_strings,
2389   .n_next_nodes = 2,
2390   .next_nodes = {
2391     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2392     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2393   },
2394 };
2395
2396 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2397                               nat44_hairpinning_fn);
2398
2399 static inline void
2400 nat44_reass_hairpinning (snat_main_t *sm,
2401                          vlib_buffer_t * b0,
2402                          ip4_header_t * ip0,
2403                          u16 sport,
2404                          u16 dport,
2405                          u32 proto0)
2406 {
2407   snat_session_key_t key0, sm0;
2408   snat_session_t * s0;
2409   clib_bihash_kv_8_8_t kv0, value0;
2410   ip_csum_t sum0;
2411   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2412   u16 new_dst_port0, old_dst_port0;
2413   udp_header_t * udp0;
2414   tcp_header_t * tcp0;
2415
2416   key0.addr = ip0->dst_address;
2417   key0.port = dport;
2418   key0.protocol = proto0;
2419   key0.fib_index = sm->outside_fib_index;
2420   kv0.key = key0.as_u64;
2421
2422   udp0 = ip4_next_header (ip0);
2423
2424   /* Check if destination is static mappings */
2425   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2426     {
2427       new_dst_addr0 = sm0.addr.as_u32;
2428       new_dst_port0 = sm0.port;
2429       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2430     }
2431   /* or active sessions */
2432   else
2433     {
2434       if (sm->num_workers > 1)
2435         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2436       else
2437         ti = sm->num_workers;
2438
2439       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2440         {
2441           si = value0.value;
2442           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2443           new_dst_addr0 = s0->in2out.addr.as_u32;
2444           new_dst_port0 = s0->in2out.port;
2445           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2446         }
2447     }
2448
2449   /* Destination is behind the same NAT, use internal address and port */
2450   if (new_dst_addr0)
2451     {
2452       old_dst_addr0 = ip0->dst_address.as_u32;
2453       ip0->dst_address.as_u32 = new_dst_addr0;
2454       sum0 = ip0->checksum;
2455       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2456                              ip4_header_t, dst_address);
2457       ip0->checksum = ip_csum_fold (sum0);
2458
2459       old_dst_port0 = dport;
2460       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2461                        ip4_is_first_fragment (ip0)))
2462         {
2463           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2464             {
2465               tcp0 = ip4_next_header (ip0);
2466               tcp0->dst = new_dst_port0;
2467               sum0 = tcp0->checksum;
2468               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2469                                      ip4_header_t, dst_address);
2470               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2471                                      ip4_header_t /* cheat */, length);
2472               tcp0->checksum = ip_csum_fold(sum0);
2473             }
2474           else
2475             {
2476               udp0->dst_port = new_dst_port0;
2477               udp0->checksum = 0;
2478             }
2479         }
2480       else
2481         {
2482           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2483             {
2484               tcp0 = ip4_next_header (ip0);
2485               sum0 = tcp0->checksum;
2486               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2487                                      ip4_header_t, dst_address);
2488               tcp0->checksum = ip_csum_fold(sum0);
2489             }
2490         }
2491     }
2492 }
2493
2494 static uword
2495 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2496                             vlib_node_runtime_t * node,
2497                             vlib_frame_t * frame)
2498 {
2499   u32 n_left_from, *from, *to_next;
2500   snat_in2out_next_t next_index;
2501   u32 pkts_processed = 0;
2502   snat_main_t *sm = &snat_main;
2503   f64 now = vlib_time_now (vm);
2504   u32 thread_index = vlib_get_thread_index ();
2505   snat_main_per_thread_data_t *per_thread_data =
2506     &sm->per_thread_data[thread_index];
2507   u32 *fragments_to_drop = 0;
2508   u32 *fragments_to_loopback = 0;
2509
2510   from = vlib_frame_vector_args (frame);
2511   n_left_from = frame->n_vectors;
2512   next_index = node->cached_next_index;
2513
2514   while (n_left_from > 0)
2515     {
2516       u32 n_left_to_next;
2517
2518       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2519
2520       while (n_left_from > 0 && n_left_to_next > 0)
2521        {
2522           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2523           vlib_buffer_t *b0;
2524           u32 next0;
2525           u8 cached0 = 0;
2526           ip4_header_t *ip0;
2527           nat_reass_ip4_t *reass0;
2528           udp_header_t * udp0;
2529           tcp_header_t * tcp0;
2530           snat_session_key_t key0;
2531           clib_bihash_kv_8_8_t kv0, value0;
2532           snat_session_t * s0 = 0;
2533           u16 old_port0, new_port0;
2534           ip_csum_t sum0;
2535
2536           /* speculatively enqueue b0 to the current next frame */
2537           bi0 = from[0];
2538           to_next[0] = bi0;
2539           from += 1;
2540           to_next += 1;
2541           n_left_from -= 1;
2542           n_left_to_next -= 1;
2543
2544           b0 = vlib_get_buffer (vm, bi0);
2545           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2546
2547           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2548           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2549                                                                sw_if_index0);
2550
2551           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2552             {
2553               next0 = SNAT_IN2OUT_NEXT_DROP;
2554               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2555               goto trace0;
2556             }
2557
2558           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2559           udp0 = ip4_next_header (ip0);
2560           tcp0 = (tcp_header_t *) udp0;
2561           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2562
2563           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2564                                                  ip0->dst_address,
2565                                                  ip0->fragment_id,
2566                                                  ip0->protocol,
2567                                                  1,
2568                                                  &fragments_to_drop);
2569
2570           if (PREDICT_FALSE (!reass0))
2571             {
2572               next0 = SNAT_IN2OUT_NEXT_DROP;
2573               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2574               goto trace0;
2575             }
2576
2577           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2578             {
2579               key0.addr = ip0->src_address;
2580               key0.port = udp0->src_port;
2581               key0.protocol = proto0;
2582               key0.fib_index = rx_fib_index0;
2583               kv0.key = key0.as_u64;
2584
2585               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2586                 {
2587                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2588                       ip0, proto0, rx_fib_index0, thread_index)))
2589                     goto trace0;
2590
2591                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2592                                      &s0, node, next0, thread_index);
2593
2594                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2595                     goto trace0;
2596
2597                   reass0->sess_index = s0 - per_thread_data->sessions;
2598                 }
2599               else
2600                 {
2601                   s0 = pool_elt_at_index (per_thread_data->sessions,
2602                                           value0.value);
2603                   reass0->sess_index = value0.value;
2604                 }
2605               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2606             }
2607           else
2608             {
2609               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2610                 {
2611                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2612                     {
2613                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2614                       next0 = SNAT_IN2OUT_NEXT_DROP;
2615                       goto trace0;
2616                     }
2617                   cached0 = 1;
2618                   goto trace0;
2619                 }
2620               s0 = pool_elt_at_index (per_thread_data->sessions,
2621                                       reass0->sess_index);
2622             }
2623
2624           old_addr0 = ip0->src_address.as_u32;
2625           ip0->src_address = s0->out2in.addr;
2626           new_addr0 = ip0->src_address.as_u32;
2627           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2628
2629           sum0 = ip0->checksum;
2630           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2631                                  ip4_header_t,
2632                                  src_address /* changed member */);
2633           ip0->checksum = ip_csum_fold (sum0);
2634
2635           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2636             {
2637               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2638                 {
2639                   old_port0 = tcp0->src_port;
2640                   tcp0->src_port = s0->out2in.port;
2641                   new_port0 = tcp0->src_port;
2642
2643                   sum0 = tcp0->checksum;
2644                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2645                                          ip4_header_t,
2646                                          dst_address /* changed member */);
2647                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2648                                          ip4_header_t /* cheat */,
2649                                          length /* changed member */);
2650                   tcp0->checksum = ip_csum_fold(sum0);
2651                 }
2652               else
2653                 {
2654                   old_port0 = udp0->src_port;
2655                   udp0->src_port = s0->out2in.port;
2656                   udp0->checksum = 0;
2657                 }
2658             }
2659
2660           /* Hairpinning */
2661           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2662                                    s0->ext_host_port, proto0);
2663
2664           /* Accounting */
2665           s0->last_heard = now;
2666           s0->total_pkts++;
2667           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2668           /* Per-user LRU list maintenance */
2669           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2670                              s0->per_user_index);
2671           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2672                               s0->per_user_list_head_index,
2673                               s0->per_user_index);
2674
2675         trace0:
2676           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2677                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2678             {
2679               nat44_in2out_reass_trace_t *t =
2680                  vlib_add_trace (vm, node, b0, sizeof (*t));
2681               t->cached = cached0;
2682               t->sw_if_index = sw_if_index0;
2683               t->next_index = next0;
2684             }
2685
2686           if (cached0)
2687             {
2688               n_left_to_next++;
2689               to_next--;
2690             }
2691           else
2692             {
2693               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2694
2695               /* verify speculative enqueue, maybe switch current next frame */
2696               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2697                                                to_next, n_left_to_next,
2698                                                bi0, next0);
2699             }
2700
2701           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2702             {
2703               from = vlib_frame_vector_args (frame);
2704               u32 len = vec_len (fragments_to_loopback);
2705               if (len <= VLIB_FRAME_SIZE)
2706                 {
2707                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2708                   n_left_from = len;
2709                   vec_reset_length (fragments_to_loopback);
2710                 }
2711               else
2712                 {
2713                   clib_memcpy (from,
2714                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2715                                sizeof (u32) * VLIB_FRAME_SIZE);
2716                   n_left_from = VLIB_FRAME_SIZE;
2717                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2718                 }
2719             }
2720        }
2721
2722       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2723     }
2724
2725   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2726                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2727                                pkts_processed);
2728
2729   nat_send_all_to_node (vm, fragments_to_drop, node,
2730                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2731                         SNAT_IN2OUT_NEXT_DROP);
2732
2733   vec_free (fragments_to_drop);
2734   vec_free (fragments_to_loopback);
2735   return frame->n_vectors;
2736 }
2737
2738 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2739   .function = nat44_in2out_reass_node_fn,
2740   .name = "nat44-in2out-reass",
2741   .vector_size = sizeof (u32),
2742   .format_trace = format_nat44_in2out_reass_trace,
2743   .type = VLIB_NODE_TYPE_INTERNAL,
2744
2745   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2746   .error_strings = snat_in2out_error_strings,
2747
2748   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2749   .next_nodes = {
2750     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2751     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2752     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2753     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2754     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2755   },
2756 };
2757
2758 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2759                               nat44_in2out_reass_node_fn);
2760
2761 /**************************/
2762 /*** deterministic mode ***/
2763 /**************************/
2764 static uword
2765 snat_det_in2out_node_fn (vlib_main_t * vm,
2766                          vlib_node_runtime_t * node,
2767                          vlib_frame_t * frame)
2768 {
2769   u32 n_left_from, * from, * to_next;
2770   snat_in2out_next_t next_index;
2771   u32 pkts_processed = 0;
2772   snat_main_t * sm = &snat_main;
2773   u32 now = (u32) vlib_time_now (vm);
2774   u32 thread_index = vlib_get_thread_index ();
2775
2776   from = vlib_frame_vector_args (frame);
2777   n_left_from = frame->n_vectors;
2778   next_index = node->cached_next_index;
2779
2780   while (n_left_from > 0)
2781     {
2782       u32 n_left_to_next;
2783
2784       vlib_get_next_frame (vm, node, next_index,
2785                            to_next, n_left_to_next);
2786
2787       while (n_left_from >= 4 && n_left_to_next >= 2)
2788         {
2789           u32 bi0, bi1;
2790           vlib_buffer_t * b0, * b1;
2791           u32 next0, next1;
2792           u32 sw_if_index0, sw_if_index1;
2793           ip4_header_t * ip0, * ip1;
2794           ip_csum_t sum0, sum1;
2795           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2796           u16 old_port0, new_port0, lo_port0, i0;
2797           u16 old_port1, new_port1, lo_port1, i1;
2798           udp_header_t * udp0, * udp1;
2799           tcp_header_t * tcp0, * tcp1;
2800           u32 proto0, proto1;
2801           snat_det_out_key_t key0, key1;
2802           snat_det_map_t * dm0, * dm1;
2803           snat_det_session_t * ses0 = 0, * ses1 = 0;
2804           u32 rx_fib_index0, rx_fib_index1;
2805           icmp46_header_t * icmp0, * icmp1;
2806
2807           /* Prefetch next iteration. */
2808           {
2809             vlib_buffer_t * p2, * p3;
2810
2811             p2 = vlib_get_buffer (vm, from[2]);
2812             p3 = vlib_get_buffer (vm, from[3]);
2813
2814             vlib_prefetch_buffer_header (p2, LOAD);
2815             vlib_prefetch_buffer_header (p3, LOAD);
2816
2817             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2818             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2819           }
2820
2821           /* speculatively enqueue b0 and b1 to the current next frame */
2822           to_next[0] = bi0 = from[0];
2823           to_next[1] = bi1 = from[1];
2824           from += 2;
2825           to_next += 2;
2826           n_left_from -= 2;
2827           n_left_to_next -= 2;
2828
2829           b0 = vlib_get_buffer (vm, bi0);
2830           b1 = vlib_get_buffer (vm, bi1);
2831
2832           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2833           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2834
2835           ip0 = vlib_buffer_get_current (b0);
2836           udp0 = ip4_next_header (ip0);
2837           tcp0 = (tcp_header_t *) udp0;
2838
2839           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2840
2841           if (PREDICT_FALSE(ip0->ttl == 1))
2842             {
2843               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2844               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2845                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2846                                            0);
2847               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2848               goto trace0;
2849             }
2850
2851           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2852
2853           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2854             {
2855               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2856               icmp0 = (icmp46_header_t *) udp0;
2857
2858               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2859                                   rx_fib_index0, node, next0, thread_index,
2860                                   &ses0, &dm0);
2861               goto trace0;
2862             }
2863
2864           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2865           if (PREDICT_FALSE(!dm0))
2866             {
2867               clib_warning("no match for internal host %U",
2868                            format_ip4_address, &ip0->src_address);
2869               next0 = SNAT_IN2OUT_NEXT_DROP;
2870               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2871               goto trace0;
2872             }
2873
2874           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2875
2876           key0.ext_host_addr = ip0->dst_address;
2877           key0.ext_host_port = tcp0->dst;
2878
2879           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2880           if (PREDICT_FALSE(!ses0))
2881             {
2882               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2883                 {
2884                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2885                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2886
2887                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2888                     continue;
2889
2890                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2891                   break;
2892                 }
2893               if (PREDICT_FALSE(!ses0))
2894                 {
2895                   /* too many sessions for user, send ICMP error packet */
2896
2897                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2898                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2899                                                ICMP4_destination_unreachable_destination_unreachable_host,
2900                                                0);
2901                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2902                   goto trace0;
2903                 }
2904             }
2905
2906           new_port0 = ses0->out.out_port;
2907
2908           old_addr0.as_u32 = ip0->src_address.as_u32;
2909           ip0->src_address.as_u32 = new_addr0.as_u32;
2910           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2911
2912           sum0 = ip0->checksum;
2913           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2914                                  ip4_header_t,
2915                                  src_address /* changed member */);
2916           ip0->checksum = ip_csum_fold (sum0);
2917
2918           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2919             {
2920               if (tcp0->flags & TCP_FLAG_SYN)
2921                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2922               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2923                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2924               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2925                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2926               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2927                 snat_det_ses_close(dm0, ses0);
2928               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2929                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2930               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2931                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2932
2933               old_port0 = tcp0->src;
2934               tcp0->src = new_port0;
2935
2936               sum0 = tcp0->checksum;
2937               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2938                                      ip4_header_t,
2939                                      dst_address /* changed member */);
2940               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2941                                      ip4_header_t /* cheat */,
2942                                      length /* changed member */);
2943               tcp0->checksum = ip_csum_fold(sum0);
2944             }
2945           else
2946             {
2947               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2948               old_port0 = udp0->src_port;
2949               udp0->src_port = new_port0;
2950               udp0->checksum = 0;
2951             }
2952
2953           switch(ses0->state)
2954             {
2955             case SNAT_SESSION_UDP_ACTIVE:
2956                 ses0->expire = now + sm->udp_timeout;
2957                 break;
2958             case SNAT_SESSION_TCP_SYN_SENT:
2959             case SNAT_SESSION_TCP_FIN_WAIT:
2960             case SNAT_SESSION_TCP_CLOSE_WAIT:
2961             case SNAT_SESSION_TCP_LAST_ACK:
2962                 ses0->expire = now + sm->tcp_transitory_timeout;
2963                 break;
2964             case SNAT_SESSION_TCP_ESTABLISHED:
2965                 ses0->expire = now + sm->tcp_established_timeout;
2966                 break;
2967             }
2968
2969         trace0:
2970           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2971                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2972             {
2973               snat_in2out_trace_t *t =
2974                  vlib_add_trace (vm, node, b0, sizeof (*t));
2975               t->is_slow_path = 0;
2976               t->sw_if_index = sw_if_index0;
2977               t->next_index = next0;
2978               t->session_index = ~0;
2979               if (ses0)
2980                 t->session_index = ses0 - dm0->sessions;
2981             }
2982
2983           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2984
2985           ip1 = vlib_buffer_get_current (b1);
2986           udp1 = ip4_next_header (ip1);
2987           tcp1 = (tcp_header_t *) udp1;
2988
2989           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2990
2991           if (PREDICT_FALSE(ip1->ttl == 1))
2992             {
2993               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2994               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2995                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2996                                            0);
2997               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2998               goto trace1;
2999             }
3000
3001           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3002
3003           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3004             {
3005               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3006               icmp1 = (icmp46_header_t *) udp1;
3007
3008               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
3009                                   rx_fib_index1, node, next1, thread_index,
3010                                   &ses1, &dm1);
3011               goto trace1;
3012             }
3013
3014           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
3015           if (PREDICT_FALSE(!dm1))
3016             {
3017               clib_warning("no match for internal host %U",
3018                            format_ip4_address, &ip0->src_address);
3019               next1 = SNAT_IN2OUT_NEXT_DROP;
3020               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3021               goto trace1;
3022             }
3023
3024           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3025
3026           key1.ext_host_addr = ip1->dst_address;
3027           key1.ext_host_port = tcp1->dst;
3028
3029           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3030           if (PREDICT_FALSE(!ses1))
3031             {
3032               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3033                 {
3034                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
3035                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3036
3037                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3038                     continue;
3039
3040                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3041                   break;
3042                 }
3043               if (PREDICT_FALSE(!ses1))
3044                 {
3045                   /* too many sessions for user, send ICMP error packet */
3046
3047                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3048                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3049                                                ICMP4_destination_unreachable_destination_unreachable_host,
3050                                                0);
3051                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3052                   goto trace1;
3053                 }
3054             }
3055
3056           new_port1 = ses1->out.out_port;
3057
3058           old_addr1.as_u32 = ip1->src_address.as_u32;
3059           ip1->src_address.as_u32 = new_addr1.as_u32;
3060           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3061
3062           sum1 = ip1->checksum;
3063           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3064                                  ip4_header_t,
3065                                  src_address /* changed member */);
3066           ip1->checksum = ip_csum_fold (sum1);
3067
3068           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3069             {
3070               if (tcp1->flags & TCP_FLAG_SYN)
3071                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3072               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3073                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3074               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3075                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3076               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3077                 snat_det_ses_close(dm1, ses1);
3078               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3079                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3080               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3081                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3082
3083               old_port1 = tcp1->src;
3084               tcp1->src = new_port1;
3085
3086               sum1 = tcp1->checksum;
3087               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3088                                      ip4_header_t,
3089                                      dst_address /* changed member */);
3090               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3091                                      ip4_header_t /* cheat */,
3092                                      length /* changed member */);
3093               tcp1->checksum = ip_csum_fold(sum1);
3094             }
3095           else
3096             {
3097               ses1->state = SNAT_SESSION_UDP_ACTIVE;
3098               old_port1 = udp1->src_port;
3099               udp1->src_port = new_port1;
3100               udp1->checksum = 0;
3101             }
3102
3103           switch(ses1->state)
3104             {
3105             case SNAT_SESSION_UDP_ACTIVE:
3106                 ses1->expire = now + sm->udp_timeout;
3107                 break;
3108             case SNAT_SESSION_TCP_SYN_SENT:
3109             case SNAT_SESSION_TCP_FIN_WAIT:
3110             case SNAT_SESSION_TCP_CLOSE_WAIT:
3111             case SNAT_SESSION_TCP_LAST_ACK:
3112                 ses1->expire = now + sm->tcp_transitory_timeout;
3113                 break;
3114             case SNAT_SESSION_TCP_ESTABLISHED:
3115                 ses1->expire = now + sm->tcp_established_timeout;
3116                 break;
3117             }
3118
3119         trace1:
3120           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3121                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3122             {
3123               snat_in2out_trace_t *t =
3124                  vlib_add_trace (vm, node, b1, sizeof (*t));
3125               t->is_slow_path = 0;
3126               t->sw_if_index = sw_if_index1;
3127               t->next_index = next1;
3128               t->session_index = ~0;
3129               if (ses1)
3130                 t->session_index = ses1 - dm1->sessions;
3131             }
3132
3133           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3134
3135           /* verify speculative enqueues, maybe switch current next frame */
3136           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3137                                            to_next, n_left_to_next,
3138                                            bi0, bi1, next0, next1);
3139          }
3140
3141       while (n_left_from > 0 && n_left_to_next > 0)
3142         {
3143           u32 bi0;
3144           vlib_buffer_t * b0;
3145           u32 next0;
3146           u32 sw_if_index0;
3147           ip4_header_t * ip0;
3148           ip_csum_t sum0;
3149           ip4_address_t new_addr0, old_addr0;
3150           u16 old_port0, new_port0, lo_port0, i0;
3151           udp_header_t * udp0;
3152           tcp_header_t * tcp0;
3153           u32 proto0;
3154           snat_det_out_key_t key0;
3155           snat_det_map_t * dm0;
3156           snat_det_session_t * ses0 = 0;
3157           u32 rx_fib_index0;
3158           icmp46_header_t * icmp0;
3159
3160           /* speculatively enqueue b0 to the current next frame */
3161           bi0 = from[0];
3162           to_next[0] = bi0;
3163           from += 1;
3164           to_next += 1;
3165           n_left_from -= 1;
3166           n_left_to_next -= 1;
3167
3168           b0 = vlib_get_buffer (vm, bi0);
3169           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3170
3171           ip0 = vlib_buffer_get_current (b0);
3172           udp0 = ip4_next_header (ip0);
3173           tcp0 = (tcp_header_t *) udp0;
3174
3175           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3176
3177           if (PREDICT_FALSE(ip0->ttl == 1))
3178             {
3179               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3180               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3181                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3182                                            0);
3183               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3184               goto trace00;
3185             }
3186
3187           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3188
3189           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3190             {
3191               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3192               icmp0 = (icmp46_header_t *) udp0;
3193
3194               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3195                                   rx_fib_index0, node, next0, thread_index,
3196                                   &ses0, &dm0);
3197               goto trace00;
3198             }
3199
3200           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3201           if (PREDICT_FALSE(!dm0))
3202             {
3203               clib_warning("no match for internal host %U",
3204                            format_ip4_address, &ip0->src_address);
3205               next0 = SNAT_IN2OUT_NEXT_DROP;
3206               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3207               goto trace00;
3208             }
3209
3210           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3211
3212           key0.ext_host_addr = ip0->dst_address;
3213           key0.ext_host_port = tcp0->dst;
3214
3215           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3216           if (PREDICT_FALSE(!ses0))
3217             {
3218               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3219                 {
3220                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3221                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3222
3223                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3224                     continue;
3225
3226                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3227                   break;
3228                 }
3229               if (PREDICT_FALSE(!ses0))
3230                 {
3231                   /* too many sessions for user, send ICMP error packet */
3232
3233                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3234                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3235                                                ICMP4_destination_unreachable_destination_unreachable_host,
3236                                                0);
3237                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3238                   goto trace00;
3239                 }
3240             }
3241
3242           new_port0 = ses0->out.out_port;
3243
3244           old_addr0.as_u32 = ip0->src_address.as_u32;
3245           ip0->src_address.as_u32 = new_addr0.as_u32;
3246           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3247
3248           sum0 = ip0->checksum;
3249           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3250                                  ip4_header_t,
3251                                  src_address /* changed member */);
3252           ip0->checksum = ip_csum_fold (sum0);
3253
3254           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3255             {
3256               if (tcp0->flags & TCP_FLAG_SYN)
3257                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3258               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3259                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3260               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3261                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3262               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3263                 snat_det_ses_close(dm0, ses0);
3264               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3265                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3266               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3267                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3268
3269               old_port0 = tcp0->src;
3270               tcp0->src = new_port0;
3271
3272               sum0 = tcp0->checksum;
3273               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3274                                      ip4_header_t,
3275                                      dst_address /* changed member */);
3276               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3277                                      ip4_header_t /* cheat */,
3278                                      length /* changed member */);
3279               tcp0->checksum = ip_csum_fold(sum0);
3280             }
3281           else
3282             {
3283               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3284               old_port0 = udp0->src_port;
3285               udp0->src_port = new_port0;
3286               udp0->checksum = 0;
3287             }
3288
3289           switch(ses0->state)
3290             {
3291             case SNAT_SESSION_UDP_ACTIVE:
3292                 ses0->expire = now + sm->udp_timeout;
3293                 break;
3294             case SNAT_SESSION_TCP_SYN_SENT:
3295             case SNAT_SESSION_TCP_FIN_WAIT:
3296             case SNAT_SESSION_TCP_CLOSE_WAIT:
3297             case SNAT_SESSION_TCP_LAST_ACK:
3298                 ses0->expire = now + sm->tcp_transitory_timeout;
3299                 break;
3300             case SNAT_SESSION_TCP_ESTABLISHED:
3301                 ses0->expire = now + sm->tcp_established_timeout;
3302                 break;
3303             }
3304
3305         trace00:
3306           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3307                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3308             {
3309               snat_in2out_trace_t *t =
3310                  vlib_add_trace (vm, node, b0, sizeof (*t));
3311               t->is_slow_path = 0;
3312               t->sw_if_index = sw_if_index0;
3313               t->next_index = next0;
3314               t->session_index = ~0;
3315               if (ses0)
3316                 t->session_index = ses0 - dm0->sessions;
3317             }
3318
3319           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3320
3321           /* verify speculative enqueue, maybe switch current next frame */
3322           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3323                                            to_next, n_left_to_next,
3324                                            bi0, next0);
3325         }
3326
3327       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3328     }
3329
3330   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3331                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3332                                pkts_processed);
3333   return frame->n_vectors;
3334 }
3335
3336 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3337   .function = snat_det_in2out_node_fn,
3338   .name = "nat44-det-in2out",
3339   .vector_size = sizeof (u32),
3340   .format_trace = format_snat_in2out_trace,
3341   .type = VLIB_NODE_TYPE_INTERNAL,
3342
3343   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3344   .error_strings = snat_in2out_error_strings,
3345
3346   .runtime_data_bytes = sizeof (snat_runtime_t),
3347
3348   .n_next_nodes = 3,
3349
3350   /* edit / add dispositions here */
3351   .next_nodes = {
3352     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3353     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3354     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3355   },
3356 };
3357
3358 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3359
3360 /**
3361  * Get address and port values to be used for ICMP packet translation
3362  * and create session if needed
3363  *
3364  * @param[in,out] sm             NAT main
3365  * @param[in,out] node           NAT node runtime
3366  * @param[in] thread_index       thread index
3367  * @param[in,out] b0             buffer containing packet to be translated
3368  * @param[out] p_proto           protocol used for matching
3369  * @param[out] p_value           address and port after NAT translation
3370  * @param[out] p_dont_translate  if packet should not be translated
3371  * @param d                      optional parameter
3372  * @param e                      optional parameter
3373  */
3374 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3375                           u32 thread_index, vlib_buffer_t *b0,
3376                           ip4_header_t *ip0, u8 *p_proto,
3377                           snat_session_key_t *p_value,
3378                           u8 *p_dont_translate, void *d, void *e)
3379 {
3380   icmp46_header_t *icmp0;
3381   u32 sw_if_index0;
3382   u32 rx_fib_index0;
3383   u8 protocol;
3384   snat_det_out_key_t key0;
3385   u8 dont_translate = 0;
3386   u32 next0 = ~0;
3387   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3388   ip4_header_t *inner_ip0;
3389   void *l4_header = 0;
3390   icmp46_header_t *inner_icmp0;
3391   snat_det_map_t * dm0 = 0;
3392   ip4_address_t new_addr0;
3393   u16 lo_port0, i0;
3394   snat_det_session_t * ses0 = 0;
3395   ip4_address_t in_addr;
3396   u16 in_port;
3397
3398   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3399   echo0 = (icmp_echo_header_t *)(icmp0+1);
3400   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3401   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3402
3403   if (!icmp_is_error_message (icmp0))
3404     {
3405       protocol = SNAT_PROTOCOL_ICMP;
3406       in_addr = ip0->src_address;
3407       in_port = echo0->identifier;
3408     }
3409   else
3410     {
3411       inner_ip0 = (ip4_header_t *)(echo0+1);
3412       l4_header = ip4_next_header (inner_ip0);
3413       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3414       in_addr = inner_ip0->dst_address;
3415       switch (protocol)
3416         {
3417         case SNAT_PROTOCOL_ICMP:
3418           inner_icmp0 = (icmp46_header_t*)l4_header;
3419           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3420           in_port = inner_echo0->identifier;
3421           break;
3422         case SNAT_PROTOCOL_UDP:
3423         case SNAT_PROTOCOL_TCP:
3424           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3425           break;
3426         default:
3427           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3428           next0 = SNAT_IN2OUT_NEXT_DROP;
3429           goto out;
3430         }
3431     }
3432
3433   dm0 = snat_det_map_by_user(sm, &in_addr);
3434   if (PREDICT_FALSE(!dm0))
3435     {
3436       clib_warning("no match for internal host %U",
3437                    format_ip4_address, &in_addr);
3438       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3439           IP_PROTOCOL_ICMP, rx_fib_index0)))
3440         {
3441           dont_translate = 1;
3442           goto out;
3443         }
3444       next0 = SNAT_IN2OUT_NEXT_DROP;
3445       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3446       goto out;
3447     }
3448
3449   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3450
3451   key0.ext_host_addr = ip0->dst_address;
3452   key0.ext_host_port = 0;
3453
3454   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3455   if (PREDICT_FALSE(!ses0))
3456     {
3457       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3458           IP_PROTOCOL_ICMP, rx_fib_index0)))
3459         {
3460           dont_translate = 1;
3461           goto out;
3462         }
3463       if (icmp0->type != ICMP4_echo_request)
3464         {
3465           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3466           next0 = SNAT_IN2OUT_NEXT_DROP;
3467           goto out;
3468         }
3469       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3470         {
3471           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3472             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3473
3474           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3475             continue;
3476
3477           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3478           break;
3479         }
3480       if (PREDICT_FALSE(!ses0))
3481         {
3482           next0 = SNAT_IN2OUT_NEXT_DROP;
3483           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3484           goto out;
3485         }
3486     }
3487
3488   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3489                     !icmp_is_error_message (icmp0)))
3490     {
3491       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3492       next0 = SNAT_IN2OUT_NEXT_DROP;
3493       goto out;
3494     }
3495
3496   u32 now = (u32) vlib_time_now (sm->vlib_main);
3497
3498   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3499   ses0->expire = now + sm->icmp_timeout;
3500
3501 out:
3502   *p_proto = protocol;
3503   if (ses0)
3504     {
3505       p_value->addr = new_addr0;
3506       p_value->fib_index = sm->outside_fib_index;
3507       p_value->port = ses0->out.out_port;
3508     }
3509   *p_dont_translate = dont_translate;
3510   if (d)
3511     *(snat_det_session_t**)d = ses0;
3512   if (e)
3513     *(snat_det_map_t**)e = dm0;
3514   return next0;
3515 }
3516
3517 /**********************/
3518 /*** worker handoff ***/
3519 /**********************/
3520 static inline uword
3521 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3522                                       vlib_node_runtime_t * node,
3523                                       vlib_frame_t * frame,
3524                                       u8 is_output)
3525 {
3526   snat_main_t *sm = &snat_main;
3527   vlib_thread_main_t *tm = vlib_get_thread_main ();
3528   u32 n_left_from, *from, *to_next = 0;
3529   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3530   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3531     = 0;
3532   vlib_frame_queue_elt_t *hf = 0;
3533   vlib_frame_t *f = 0;
3534   int i;
3535   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3536   u32 next_worker_index = 0;
3537   u32 current_worker_index = ~0;
3538   u32 thread_index = vlib_get_thread_index ();
3539   u32 fq_index;
3540   u32 to_node_index;
3541
3542   ASSERT (vec_len (sm->workers));
3543
3544   if (is_output)
3545     {
3546       fq_index = sm->fq_in2out_output_index;
3547       to_node_index = sm->in2out_output_node_index;
3548     }
3549   else
3550     {
3551       fq_index = sm->fq_in2out_index;
3552       to_node_index = sm->in2out_node_index;
3553     }
3554
3555   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3556     {
3557       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3558
3559       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3560                                sm->first_worker_index + sm->num_workers - 1,
3561                                (vlib_frame_queue_t *) (~0));
3562     }
3563
3564   from = vlib_frame_vector_args (frame);
3565   n_left_from = frame->n_vectors;
3566
3567   while (n_left_from > 0)
3568     {
3569       u32 bi0;
3570       vlib_buffer_t *b0;
3571       u32 sw_if_index0;
3572       u32 rx_fib_index0;
3573       ip4_header_t * ip0;
3574       u8 do_handoff;
3575
3576       bi0 = from[0];
3577       from += 1;
3578       n_left_from -= 1;
3579
3580       b0 = vlib_get_buffer (vm, bi0);
3581
3582       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3583       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3584
3585       ip0 = vlib_buffer_get_current (b0);
3586
3587       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3588
3589       if (PREDICT_FALSE (next_worker_index != thread_index))
3590         {
3591           do_handoff = 1;
3592
3593           if (next_worker_index != current_worker_index)
3594             {
3595               if (hf)
3596                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3597
3598               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3599                                                       next_worker_index,
3600                                                       handoff_queue_elt_by_worker_index);
3601
3602               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3603               to_next_worker = &hf->buffer_index[hf->n_vectors];
3604               current_worker_index = next_worker_index;
3605             }
3606
3607           /* enqueue to correct worker thread */
3608           to_next_worker[0] = bi0;
3609           to_next_worker++;
3610           n_left_to_next_worker--;
3611
3612           if (n_left_to_next_worker == 0)
3613             {
3614               hf->n_vectors = VLIB_FRAME_SIZE;
3615               vlib_put_frame_queue_elt (hf);
3616               current_worker_index = ~0;
3617               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3618               hf = 0;
3619             }
3620         }
3621       else
3622         {
3623           do_handoff = 0;
3624           /* if this is 1st frame */
3625           if (!f)
3626             {
3627               f = vlib_get_frame_to_node (vm, to_node_index);
3628               to_next = vlib_frame_vector_args (f);
3629             }
3630
3631           to_next[0] = bi0;
3632           to_next += 1;
3633           f->n_vectors++;
3634         }
3635
3636       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3637                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3638         {
3639           snat_in2out_worker_handoff_trace_t *t =
3640             vlib_add_trace (vm, node, b0, sizeof (*t));
3641           t->next_worker_index = next_worker_index;
3642           t->do_handoff = do_handoff;
3643         }
3644     }
3645
3646   if (f)
3647     vlib_put_frame_to_node (vm, to_node_index, f);
3648
3649   if (hf)
3650     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3651
3652   /* Ship frames to the worker nodes */
3653   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3654     {
3655       if (handoff_queue_elt_by_worker_index[i])
3656         {
3657           hf = handoff_queue_elt_by_worker_index[i];
3658           /*
3659            * It works better to let the handoff node
3660            * rate-adapt, always ship the handoff queue element.
3661            */
3662           if (1 || hf->n_vectors == hf->last_n_vectors)
3663             {
3664               vlib_put_frame_queue_elt (hf);
3665               handoff_queue_elt_by_worker_index[i] = 0;
3666             }
3667           else
3668             hf->last_n_vectors = hf->n_vectors;
3669         }
3670       congested_handoff_queue_by_worker_index[i] =
3671         (vlib_frame_queue_t *) (~0);
3672     }
3673   hf = 0;
3674   current_worker_index = ~0;
3675   return frame->n_vectors;
3676 }
3677
3678 static uword
3679 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3680                                vlib_node_runtime_t * node,
3681                                vlib_frame_t * frame)
3682 {
3683   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3684 }
3685
3686 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3687   .function = snat_in2out_worker_handoff_fn,
3688   .name = "nat44-in2out-worker-handoff",
3689   .vector_size = sizeof (u32),
3690   .format_trace = format_snat_in2out_worker_handoff_trace,
3691   .type = VLIB_NODE_TYPE_INTERNAL,
3692
3693   .n_next_nodes = 1,
3694
3695   .next_nodes = {
3696     [0] = "error-drop",
3697   },
3698 };
3699
3700 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3701                               snat_in2out_worker_handoff_fn);
3702
3703 static uword
3704 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3705                                       vlib_node_runtime_t * node,
3706                                       vlib_frame_t * frame)
3707 {
3708   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3709 }
3710
3711 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3712   .function = snat_in2out_output_worker_handoff_fn,
3713   .name = "nat44-in2out-output-worker-handoff",
3714   .vector_size = sizeof (u32),
3715   .format_trace = format_snat_in2out_worker_handoff_trace,
3716   .type = VLIB_NODE_TYPE_INTERNAL,
3717
3718   .n_next_nodes = 1,
3719
3720   .next_nodes = {
3721     [0] = "error-drop",
3722   },
3723 };
3724
3725 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3726                               snat_in2out_output_worker_handoff_fn);
3727
3728 static_always_inline int
3729 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3730 {
3731   snat_address_t * ap;
3732   clib_bihash_kv_8_8_t kv, value;
3733   snat_session_key_t m_key;
3734
3735   vec_foreach (ap, sm->addresses)
3736     {
3737       if (ap->addr.as_u32 == dst_addr->as_u32)
3738         return 1;
3739     }
3740
3741   m_key.addr.as_u32 = dst_addr->as_u32;
3742   m_key.fib_index = sm->outside_fib_index;
3743   m_key.port = 0;
3744   m_key.protocol = 0;
3745   kv.key = m_key.as_u64;
3746   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3747     return 1;
3748
3749   return 0;
3750 }
3751
3752 static uword
3753 snat_hairpin_dst_fn (vlib_main_t * vm,
3754                      vlib_node_runtime_t * node,
3755                      vlib_frame_t * frame)
3756 {
3757   u32 n_left_from, * from, * to_next;
3758   snat_in2out_next_t next_index;
3759   u32 pkts_processed = 0;
3760   snat_main_t * sm = &snat_main;
3761
3762   from = vlib_frame_vector_args (frame);
3763   n_left_from = frame->n_vectors;
3764   next_index = node->cached_next_index;
3765
3766   while (n_left_from > 0)
3767     {
3768       u32 n_left_to_next;
3769
3770       vlib_get_next_frame (vm, node, next_index,
3771                            to_next, n_left_to_next);
3772
3773       while (n_left_from > 0 && n_left_to_next > 0)
3774         {
3775           u32 bi0;
3776           vlib_buffer_t * b0;
3777           u32 next0;
3778           ip4_header_t * ip0;
3779           u32 proto0;
3780
3781           /* speculatively enqueue b0 to the current next frame */
3782           bi0 = from[0];
3783           to_next[0] = bi0;
3784           from += 1;
3785           to_next += 1;
3786           n_left_from -= 1;
3787           n_left_to_next -= 1;
3788
3789           b0 = vlib_get_buffer (vm, bi0);
3790           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3791           ip0 = vlib_buffer_get_current (b0);
3792
3793           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3794
3795           vnet_buffer (b0)->snat.flags = 0;
3796           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3797             {
3798               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3799                 {
3800                   udp_header_t * udp0 = ip4_next_header (ip0);
3801                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3802
3803                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3804                 }
3805               else if (proto0 == SNAT_PROTOCOL_ICMP)
3806                 {
3807                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3808
3809                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3810                 }
3811               else
3812                 {
3813                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3814                 }
3815
3816               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3817             }
3818
3819           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3820
3821           /* verify speculative enqueue, maybe switch current next frame */
3822           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3823                                            to_next, n_left_to_next,
3824                                            bi0, next0);
3825          }
3826
3827       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3828     }
3829
3830   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3831                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3832                                pkts_processed);
3833   return frame->n_vectors;
3834 }
3835
3836 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3837   .function = snat_hairpin_dst_fn,
3838   .name = "nat44-hairpin-dst",
3839   .vector_size = sizeof (u32),
3840   .type = VLIB_NODE_TYPE_INTERNAL,
3841   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3842   .error_strings = snat_in2out_error_strings,
3843   .n_next_nodes = 2,
3844   .next_nodes = {
3845     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3846     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3847   },
3848 };
3849
3850 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3851                               snat_hairpin_dst_fn);
3852
3853 static uword
3854 snat_hairpin_src_fn (vlib_main_t * vm,
3855                      vlib_node_runtime_t * node,
3856                      vlib_frame_t * frame)
3857 {
3858   u32 n_left_from, * from, * to_next;
3859   snat_in2out_next_t next_index;
3860   u32 pkts_processed = 0;
3861   snat_main_t *sm = &snat_main;
3862
3863   from = vlib_frame_vector_args (frame);
3864   n_left_from = frame->n_vectors;
3865   next_index = node->cached_next_index;
3866
3867   while (n_left_from > 0)
3868     {
3869       u32 n_left_to_next;
3870
3871       vlib_get_next_frame (vm, node, next_index,
3872                            to_next, n_left_to_next);
3873
3874       while (n_left_from > 0 && n_left_to_next > 0)
3875         {
3876           u32 bi0;
3877           vlib_buffer_t * b0;
3878           u32 next0;
3879           snat_interface_t *i;
3880           u32 sw_if_index0;
3881
3882           /* speculatively enqueue b0 to the current next frame */
3883           bi0 = from[0];
3884           to_next[0] = bi0;
3885           from += 1;
3886           to_next += 1;
3887           n_left_from -= 1;
3888           n_left_to_next -= 1;
3889
3890           b0 = vlib_get_buffer (vm, bi0);
3891           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3892           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3893
3894           pool_foreach (i, sm->output_feature_interfaces,
3895           ({
3896             /* Only packets from NAT inside interface */
3897             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3898               {
3899                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3900                                     SNAT_FLAG_HAIRPINNING))
3901                   {
3902                     if (PREDICT_TRUE (sm->num_workers > 1))
3903                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3904                     else
3905                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3906                   }
3907                 break;
3908               }
3909           }));
3910
3911           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3912
3913           /* verify speculative enqueue, maybe switch current next frame */
3914           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3915                                            to_next, n_left_to_next,
3916                                            bi0, next0);
3917          }
3918
3919       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3920     }
3921
3922   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3923                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3924                                pkts_processed);
3925   return frame->n_vectors;
3926 }
3927
3928 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3929   .function = snat_hairpin_src_fn,
3930   .name = "nat44-hairpin-src",
3931   .vector_size = sizeof (u32),
3932   .type = VLIB_NODE_TYPE_INTERNAL,
3933   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3934   .error_strings = snat_in2out_error_strings,
3935   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3936   .next_nodes = {
3937      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3938      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3939      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3940      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3941   },
3942 };
3943
3944 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3945                               snat_hairpin_src_fn);
3946
3947 static uword
3948 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3949                                 vlib_node_runtime_t * node,
3950                                 vlib_frame_t * frame)
3951 {
3952   u32 n_left_from, * from, * to_next;
3953   snat_in2out_next_t next_index;
3954   u32 pkts_processed = 0;
3955   snat_main_t * sm = &snat_main;
3956   u32 stats_node_index;
3957
3958   stats_node_index = snat_in2out_fast_node.index;
3959
3960   from = vlib_frame_vector_args (frame);
3961   n_left_from = frame->n_vectors;
3962   next_index = node->cached_next_index;
3963
3964   while (n_left_from > 0)
3965     {
3966       u32 n_left_to_next;
3967
3968       vlib_get_next_frame (vm, node, next_index,
3969                            to_next, n_left_to_next);
3970
3971       while (n_left_from > 0 && n_left_to_next > 0)
3972         {
3973           u32 bi0;
3974           vlib_buffer_t * b0;
3975           u32 next0;
3976           u32 sw_if_index0;
3977           ip4_header_t * ip0;
3978           ip_csum_t sum0;
3979           u32 new_addr0, old_addr0;
3980           u16 old_port0, new_port0;
3981           udp_header_t * udp0;
3982           tcp_header_t * tcp0;
3983           icmp46_header_t * icmp0;
3984           snat_session_key_t key0, sm0;
3985           u32 proto0;
3986           u32 rx_fib_index0;
3987
3988           /* speculatively enqueue b0 to the current next frame */
3989           bi0 = from[0];
3990           to_next[0] = bi0;
3991           from += 1;
3992           to_next += 1;
3993           n_left_from -= 1;
3994           n_left_to_next -= 1;
3995
3996           b0 = vlib_get_buffer (vm, bi0);
3997           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3998
3999           ip0 = vlib_buffer_get_current (b0);
4000           udp0 = ip4_next_header (ip0);
4001           tcp0 = (tcp_header_t *) udp0;
4002           icmp0 = (icmp46_header_t *) udp0;
4003
4004           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4005           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4006
4007           if (PREDICT_FALSE(ip0->ttl == 1))
4008             {
4009               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4010               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4011                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4012                                            0);
4013               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4014               goto trace0;
4015             }
4016
4017           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4018
4019           if (PREDICT_FALSE (proto0 == ~0))
4020               goto trace0;
4021
4022           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4023             {
4024               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4025                                   rx_fib_index0, node, next0, ~0, 0, 0);
4026               goto trace0;
4027             }
4028
4029           key0.addr = ip0->src_address;
4030           key0.protocol = proto0;
4031           key0.port = udp0->src_port;
4032           key0.fib_index = rx_fib_index0;
4033
4034           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
4035             {
4036               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4037               next0= SNAT_IN2OUT_NEXT_DROP;
4038               goto trace0;
4039             }
4040
4041           new_addr0 = sm0.addr.as_u32;
4042           new_port0 = sm0.port;
4043           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4044           old_addr0 = ip0->src_address.as_u32;
4045           ip0->src_address.as_u32 = new_addr0;
4046
4047           sum0 = ip0->checksum;
4048           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4049                                  ip4_header_t,
4050                                  src_address /* changed member */);
4051           ip0->checksum = ip_csum_fold (sum0);
4052
4053           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4054             {
4055               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4056                 {
4057                   old_port0 = tcp0->src_port;
4058                   tcp0->src_port = new_port0;
4059
4060                   sum0 = tcp0->checksum;
4061                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4062                                          ip4_header_t,
4063                                          dst_address /* changed member */);
4064                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4065                                          ip4_header_t /* cheat */,
4066                                          length /* changed member */);
4067                   tcp0->checksum = ip_csum_fold(sum0);
4068                 }
4069               else
4070                 {
4071                   old_port0 = udp0->src_port;
4072                   udp0->src_port = new_port0;
4073                   udp0->checksum = 0;
4074                 }
4075             }
4076           else
4077             {
4078               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4079                 {
4080                   sum0 = tcp0->checksum;
4081                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4082                                          ip4_header_t,
4083                                          dst_address /* changed member */);
4084                   tcp0->checksum = ip_csum_fold(sum0);
4085                 }
4086             }
4087
4088           /* Hairpinning */
4089           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4090
4091         trace0:
4092           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4093                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4094             {
4095               snat_in2out_trace_t *t =
4096                  vlib_add_trace (vm, node, b0, sizeof (*t));
4097               t->sw_if_index = sw_if_index0;
4098               t->next_index = next0;
4099             }
4100
4101           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4102
4103           /* verify speculative enqueue, maybe switch current next frame */
4104           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4105                                            to_next, n_left_to_next,
4106                                            bi0, next0);
4107         }
4108
4109       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4110     }
4111
4112   vlib_node_increment_counter (vm, stats_node_index,
4113                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4114                                pkts_processed);
4115   return frame->n_vectors;
4116 }
4117
4118
4119 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
4120   .function = snat_in2out_fast_static_map_fn,
4121   .name = "nat44-in2out-fast",
4122   .vector_size = sizeof (u32),
4123   .format_trace = format_snat_in2out_fast_trace,
4124   .type = VLIB_NODE_TYPE_INTERNAL,
4125
4126   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4127   .error_strings = snat_in2out_error_strings,
4128
4129   .runtime_data_bytes = sizeof (snat_runtime_t),
4130
4131   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4132
4133   /* edit / add dispositions here */
4134   .next_nodes = {
4135     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4136     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4137     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4138     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4139     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4140   },
4141 };
4142
4143 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);