src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37   u32 is_slow_path;
  38 } snat_in2out_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_in2out_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  51   char * tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  66
  67   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  68               t->sw_if_index, t->next_index);
  69
  70   return s;
  71 }
  72
  73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   snat_in2out_worker_handoff_trace_t * t =
  78     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  79   char * m;
  80
  81   m = t->do_handoff ? "next worker" : "same worker";
  82   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  83
  84   return s;
  85 }
  86
  87 typedef struct {
  88   u32 sw_if_index;
  89   u32 next_index;
  90   u8 cached;
  91 } nat44_in2out_reass_trace_t;
  92
  93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  94 {
  95   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  96   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  97   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  98
  99   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 100               t->sw_if_index, t->next_index,
 101               t->cached ? "cached" : "translated");
 102
 103   return s;
 104 }
 105
 106 vlib_node_registration_t snat_in2out_node;
 107 vlib_node_registration_t snat_in2out_slowpath_node;
 108 vlib_node_registration_t snat_in2out_fast_node;
 109 vlib_node_registration_t snat_in2out_worker_handoff_node;
 110 vlib_node_registration_t snat_det_in2out_node;
 111 vlib_node_registration_t snat_in2out_output_node;
 112 vlib_node_registration_t snat_in2out_output_slowpath_node;
 113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 114 vlib_node_registration_t snat_hairpin_dst_node;
 115 vlib_node_registration_t snat_hairpin_src_node;
 116 vlib_node_registration_t nat44_hairpinning_node;
 117 vlib_node_registration_t nat44_in2out_reass_node;
 118
 119
 120 #define foreach_snat_in2out_error                       \
 121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 122 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 123 _(OUT_OF_PORTS, "Out of ports")                         \
 124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 125 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 126 _(NO_TRANSLATION, "No translation")                     \
 127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 128 _(DROP_FRAGMENT, "Drop fragment")                       \
 129 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 131
 132 typedef enum {
 133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 134   foreach_snat_in2out_error
 135 #undef _
 136   SNAT_IN2OUT_N_ERROR,
 137 } snat_in2out_error_t;
 138
 139 static char * snat_in2out_error_strings[] = {
 140 #define _(sym,string) string,
 141   foreach_snat_in2out_error
 142 #undef _
 143 };
 144
 145 typedef enum {
 146   SNAT_IN2OUT_NEXT_LOOKUP,
 147   SNAT_IN2OUT_NEXT_DROP,
 148   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 149   SNAT_IN2OUT_NEXT_SLOW_PATH,
 150   SNAT_IN2OUT_NEXT_REASS,
 151   SNAT_IN2OUT_N_NEXT,
 152 } snat_in2out_next_t;
 153
 154 typedef enum {
 155   SNAT_HAIRPIN_SRC_NEXT_DROP,
 156   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 157   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 158   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 159   SNAT_HAIRPIN_SRC_N_NEXT,
 160 } snat_hairpin_next_t;
 161
 162 /**
 163  * @brief Check if packet should be translated
 164  *
 165  * Packets aimed at outside interface and external address with active session
 166  * should be translated.
 167  *
 168  * @param sm            NAT main
 169  * @param rt            NAT runtime data
 170  * @param sw_if_index0  index of the inside interface
 171  * @param ip0           IPv4 header
 172  * @param proto0        NAT protocol
 173  * @param rx_fib_index0 RX FIB index
 174  *
 175  * @returns 0 if packet should be translated otherwise 1
 176  */
 177 static inline int
 178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 179                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                          u32 rx_fib_index0)
 181 {
 182   if (sm->out2in_dpo)
 183     return 0;
 184
 185   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 186   fib_prefix_t pfx = {
 187     .fp_proto = FIB_PROTOCOL_IP4,
 188     .fp_len = 32,
 189     .fp_addr = {
 190         .ip4.as_u32 = ip0->dst_address.as_u32,
 191     },
 192   };
 193
 194   /* Don't NAT packet aimed at the intfc address */
 195   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 196                                       ip0->dst_address.as_u32)))
 197     return 1;
 198
 199   fei = fib_table_lookup (rx_fib_index0, &pfx);
 200   if (FIB_NODE_INDEX_INVALID != fei)
 201     {
 202       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 203       if (sw_if_index == ~0)
 204         {
 205           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 206           if (FIB_NODE_INDEX_INVALID != fei)
 207             sw_if_index = fib_entry_get_resolving_interface (fei);
 208         }
 209       snat_interface_t *i;
 210       pool_foreach (i, sm->interfaces,
 211       ({
 212         /* NAT packet aimed at outside interface */
 213         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 214           return 0;
 215       }));
 216     }
 217
 218   return 1;
 219 }
 220
 221 static inline int
 222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 223                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 224                     u32 rx_fib_index0, u32 thread_index)
 225 {
 226   udp_header_t * udp0 = ip4_next_header (ip0);
 227   snat_session_key_t key0, sm0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229
 230   key0.addr = ip0->dst_address;
 231   key0.port = udp0->dst_port;
 232   key0.protocol = proto0;
 233   key0.fib_index = sm->outside_fib_index;
 234   kv0.key = key0.as_u64;
 235
 236   /* NAT packet aimed at external address if */
 237   /* has active sessions */
 238   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 239                               &value0))
 240     {
 241       /* or is static mappings */
 242       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
 243         return 0;
 244     }
 245   else
 246     return 0;
 247
 248   if (sm->forwarding_enabled)
 249     return 1;
 250
 251   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 252                                  rx_fib_index0);
 253 }
 254
 255 static inline int
 256 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 257                                   u32 proto0, u16 src_port, u16 dst_port,
 258                                   u32 thread_index, u32 sw_if_index)
 259 {
 260   snat_session_key_t key0;
 261   clib_bihash_kv_8_8_t kv0, value0;
 262   snat_interface_t *i;
 263
 264   /* src NAT check */
 265   key0.addr = ip0->src_address;
 266   key0.port = src_port;
 267   key0.protocol = proto0;
 268   key0.fib_index = sm->outside_fib_index;
 269   kv0.key = key0.as_u64;
 270
 271   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 272                                &value0))
 273     return 1;
 274
 275   /* dst NAT check */
 276   key0.addr = ip0->dst_address;
 277   key0.port = dst_port;
 278   key0.protocol = proto0;
 279   key0.fib_index = sm->inside_fib_index;
 280   kv0.key = key0.as_u64;
 281   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 282                                &value0))
 283   {
 284     /* hairpinning */
 285     pool_foreach (i, sm->output_feature_interfaces,
 286     ({
 287       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 288         return 0;
 289     }));
 290     return 1;
 291   }
 292
 293   return 0;
 294 }
 295
 296 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 297                       ip4_header_t * ip0,
 298                       u32 rx_fib_index0,
 299                       snat_session_key_t * key0,
 300                       snat_session_t ** sessionp,
 301                       vlib_node_runtime_t * node,
 302                       u32 next0,
 303                       u32 thread_index)
 304 {
 305   snat_user_t *u;
 306   snat_session_t *s;
 307   clib_bihash_kv_8_8_t kv0;
 308   snat_session_key_t key1;
 309   u32 address_index = ~0;
 310   u32 outside_fib_index;
 311   uword * p;
 312   udp_header_t * udp0 = ip4_next_header (ip0);
 313
 314   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 315     {
 316       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 317       nat_ipfix_logging_max_sessions(sm->max_translations);
 318       return SNAT_IN2OUT_NEXT_DROP;
 319     }
 320
 321   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 322   if (! p)
 323     {
 324       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 325       return SNAT_IN2OUT_NEXT_DROP;
 326     }
 327   outside_fib_index = p[0];
 328
 329   key1.protocol = key0->protocol;
 330
 331   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 332                               thread_index);
 333   if (!u)
 334     {
 335       clib_warning ("create NAT user failed");
 336       return SNAT_IN2OUT_NEXT_DROP;
 337     }
 338
 339   /* First try to match static mapping by local address and port */
 340   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0))
 341     {
 342       /* Try to create dynamic translation */
 343       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 344                                                thread_index, &key1,
 345                                                &address_index,
 346                                                sm->port_per_thread,
 347                                                sm->per_thread_data[thread_index].snat_thread_index))
 348         {
 349           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 350           return SNAT_IN2OUT_NEXT_DROP;
 351         }
 352       u->nsessions++;
 353     }
 354   else
 355     {
 356       u->nstaticsessions++;
 357     }
 358
 359   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 360   if (!s)
 361     {
 362       clib_warning ("create NAT session failed");
 363       return SNAT_IN2OUT_NEXT_DROP;
 364     }
 365
 366   if (address_index == ~0)
 367     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 368   s->outside_address_index = address_index;
 369   s->in2out = *key0;
 370   s->out2in = key1;
 371   s->out2in.protocol = key0->protocol;
 372   s->out2in.fib_index = outside_fib_index;
 373   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 374   s->ext_host_port = udp0->dst_port;
 375   *sessionp = s;
 376
 377   /* Add to translation hashes */
 378   kv0.key = s->in2out.as_u64;
 379   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 380   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 381                                1 /* is_add */))
 382       clib_warning ("in2out key add failed");
 383
 384   kv0.key = s->out2in.as_u64;
 385   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 386
 387   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 388                                1 /* is_add */))
 389       clib_warning ("out2in key add failed");
 390
 391   /* log NAT event */
 392   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 393                                       s->out2in.addr.as_u32,
 394                                       s->in2out.protocol,
 395                                       s->in2out.port,
 396                                       s->out2in.port,
 397                                       s->in2out.fib_index);
 398   return next0;
 399 }
 400
 401 static_always_inline
 402 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 403                                  snat_session_key_t *p_key0)
 404 {
 405   icmp46_header_t *icmp0;
 406   snat_session_key_t key0;
 407   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 408   ip4_header_t *inner_ip0 = 0;
 409   void *l4_header = 0;
 410   icmp46_header_t *inner_icmp0;
 411
 412   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 413   echo0 = (icmp_echo_header_t *)(icmp0+1);
 414
 415   if (!icmp_is_error_message (icmp0))
 416     {
 417       key0.protocol = SNAT_PROTOCOL_ICMP;
 418       key0.addr = ip0->src_address;
 419       key0.port = echo0->identifier;
 420     }
 421   else
 422     {
 423       inner_ip0 = (ip4_header_t *)(echo0+1);
 424       l4_header = ip4_next_header (inner_ip0);
 425       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 426       key0.addr = inner_ip0->dst_address;
 427       switch (key0.protocol)
 428         {
 429         case SNAT_PROTOCOL_ICMP:
 430           inner_icmp0 = (icmp46_header_t*)l4_header;
 431           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 432           key0.port = inner_echo0->identifier;
 433           break;
 434         case SNAT_PROTOCOL_UDP:
 435         case SNAT_PROTOCOL_TCP:
 436           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 437           break;
 438         default:
 439           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 440         }
 441     }
 442   *p_key0 = key0;
 443   return -1; /* success */
 444 }
 445
 446 static_always_inline int
 447 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 448 {
 449   icmp46_header_t *icmp0;
 450   nat_ed_ses_key_t key0;
 451   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 452   ip4_header_t *inner_ip0 = 0;
 453   void *l4_header = 0;
 454   icmp46_header_t *inner_icmp0;
 455
 456   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 457   echo0 = (icmp_echo_header_t *)(icmp0+1);
 458
 459   if (!icmp_is_error_message (icmp0))
 460     {
 461       key0.proto = IP_PROTOCOL_ICMP;
 462       key0.l_addr = ip0->src_address;
 463       key0.r_addr = ip0->dst_address;
 464       key0.l_port = key0.r_port = echo0->identifier;
 465     }
 466   else
 467     {
 468       inner_ip0 = (ip4_header_t *)(echo0+1);
 469       l4_header = ip4_next_header (inner_ip0);
 470       key0.proto = inner_ip0->protocol;
 471       key0.r_addr = inner_ip0->src_address;
 472       key0.l_addr = inner_ip0->dst_address;
 473       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 474         {
 475         case SNAT_PROTOCOL_ICMP:
 476           inner_icmp0 = (icmp46_header_t*)l4_header;
 477           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 478           key0.r_port = key0.l_port = inner_echo0->identifier;
 479           break;
 480         case SNAT_PROTOCOL_UDP:
 481         case SNAT_PROTOCOL_TCP:
 482           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 483           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
 484           break;
 485         default:
 486           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 487         }
 488     }
 489   *p_key0 = key0;
 490   return 0;
 491 }
 492
 493 static inline int
 494 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip)
 495 {
 496   nat_ed_ses_key_t key;
 497   clib_bihash_kv_16_8_t kv, value;
 498   udp_header_t *udp;
 499
 500   if (!sm->forwarding_enabled)
 501     return 0;
 502
 503   if (ip->protocol == IP_PROTOCOL_ICMP)
 504     {
 505       if (icmp_get_ed_key (ip, &key))
 506         return 0;
 507     }
 508   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 509     {
 510       udp = ip4_next_header(ip);
 511       key.l_addr = ip->src_address;
 512       key.r_addr = ip->dst_address;
 513       key.proto = ip->protocol;
 514       key.r_port = udp->dst_port;
 515       key.l_port = udp->src_port;
 516     }
 517   else
 518     {
 519       key.l_addr = ip->src_address;
 520       key.r_addr = ip->dst_address;
 521       key.proto = ip->protocol;
 522       key.l_port = key.r_port = 0;
 523     }
 524   key.fib_index = 0;
 525   kv.key[0] = key.as_u64[0];
 526   kv.key[1] = key.as_u64[1];
 527
 528   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 529     return value.value == ~0ULL;
 530
 531   return 0;
 532 }
 533
 534 /**
 535  * Get address and port values to be used for ICMP packet translation
 536  * and create session if needed
 537  *
 538  * @param[in,out] sm             NAT main
 539  * @param[in,out] node           NAT node runtime
 540  * @param[in] thread_index       thread index
 541  * @param[in,out] b0             buffer containing packet to be translated
 542  * @param[out] p_proto           protocol used for matching
 543  * @param[out] p_value           address and port after NAT translation
 544  * @param[out] p_dont_translate  if packet should not be translated
 545  * @param d                      optional parameter
 546  * @param e                      optional parameter
 547  */
 548 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 549                            u32 thread_index, vlib_buffer_t *b0,
 550                            ip4_header_t *ip0, u8 *p_proto,
 551                            snat_session_key_t *p_value,
 552                            u8 *p_dont_translate, void *d, void *e)
 553 {
 554   icmp46_header_t *icmp0;
 555   u32 sw_if_index0;
 556   u32 rx_fib_index0;
 557   snat_session_key_t key0;
 558   snat_session_t *s0 = 0;
 559   u8 dont_translate = 0;
 560   clib_bihash_kv_8_8_t kv0, value0;
 561   u32 next0 = ~0;
 562   int err;
 563
 564   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 565   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 566   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 567
 568   err = icmp_get_key (ip0, &key0);
 569   if (err != -1)
 570     {
 571       b0->error = node->errors[err];
 572       next0 = SNAT_IN2OUT_NEXT_DROP;
 573       goto out;
 574     }
 575   key0.fib_index = rx_fib_index0;
 576
 577   kv0.key = key0.as_u64;
 578
 579   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 580                               &value0))
 581     {
 582       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 583         {
 584           if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
 585               ip0, SNAT_PROTOCOL_ICMP, key0.port, key0.port, thread_index, sw_if_index0)))
 586             {
 587               dont_translate = 1;
 588               goto out;
 589             }
 590         }
 591       else
 592         {
 593           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 594               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 595             {
 596               dont_translate = 1;
 597               goto out;
 598             }
 599         }
 600
 601       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 602         {
 603           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 604           next0 = SNAT_IN2OUT_NEXT_DROP;
 605           goto out;
 606         }
 607
 608       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 609                          &s0, node, next0, thread_index);
 610
 611       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 612         goto out;
 613     }
 614   else
 615     {
 616       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 617                         icmp0->type != ICMP4_echo_reply &&
 618                         !icmp_is_error_message (icmp0)))
 619         {
 620           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 621           next0 = SNAT_IN2OUT_NEXT_DROP;
 622           goto out;
 623         }
 624
 625       if (PREDICT_FALSE (value0.value == ~0ULL))
 626         {
 627           nat_ed_ses_key_t key;
 628           clib_bihash_kv_16_8_t s_kv, s_value;
 629
 630           key.as_u64[0] = 0;
 631           key.as_u64[1] = 0;
 632           if (icmp_get_ed_key (ip0, &key))
 633             {
 634               b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 635               next0 = SNAT_IN2OUT_NEXT_DROP;
 636               goto out;
 637             }
 638           key.fib_index = rx_fib_index0;
 639           s_kv.key[0] = key.as_u64[0];
 640           s_kv.key[1] = key.as_u64[1];
 641           if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 642             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 643                                     s_value.value);
 644           else
 645            {
 646               next0 = SNAT_IN2OUT_NEXT_DROP;
 647               goto out;
 648            }
 649         }
 650       else
 651         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 652                                 value0.value);
 653     }
 654
 655 out:
 656   *p_proto = key0.protocol;
 657   if (s0)
 658     *p_value = s0->out2in;
 659   *p_dont_translate = dont_translate;
 660   if (d)
 661     *(snat_session_t**)d = s0;
 662   return next0;
 663 }
 664
 665 /**
 666  * Get address and port values to be used for ICMP packet translation
 667  *
 668  * @param[in] sm                 NAT main
 669  * @param[in,out] node           NAT node runtime
 670  * @param[in] thread_index       thread index
 671  * @param[in,out] b0             buffer containing packet to be translated
 672  * @param[out] p_proto           protocol used for matching
 673  * @param[out] p_value           address and port after NAT translation
 674  * @param[out] p_dont_translate  if packet should not be translated
 675  * @param d                      optional parameter
 676  * @param e                      optional parameter
 677  */
 678 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 679                            u32 thread_index, vlib_buffer_t *b0,
 680                            ip4_header_t *ip0, u8 *p_proto,
 681                            snat_session_key_t *p_value,
 682                            u8 *p_dont_translate, void *d, void *e)
 683 {
 684   icmp46_header_t *icmp0;
 685   u32 sw_if_index0;
 686   u32 rx_fib_index0;
 687   snat_session_key_t key0;
 688   snat_session_key_t sm0;
 689   u8 dont_translate = 0;
 690   u8 is_addr_only;
 691   u32 next0 = ~0;
 692   int err;
 693
 694   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 695   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 696   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 697
 698   err = icmp_get_key (ip0, &key0);
 699   if (err != -1)
 700     {
 701       b0->error = node->errors[err];
 702       next0 = SNAT_IN2OUT_NEXT_DROP;
 703       goto out2;
 704     }
 705   key0.fib_index = rx_fib_index0;
 706
 707   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0))
 708     {
 709       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 710           IP_PROTOCOL_ICMP, rx_fib_index0)))
 711         {
 712           dont_translate = 1;
 713           goto out;
 714         }
 715
 716       if (icmp_is_error_message (icmp0))
 717         {
 718           next0 = SNAT_IN2OUT_NEXT_DROP;
 719           goto out;
 720         }
 721
 722       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 723       next0 = SNAT_IN2OUT_NEXT_DROP;
 724       goto out;
 725     }
 726
 727   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 728                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 729                     !icmp_is_error_message (icmp0)))
 730     {
 731       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 732       next0 = SNAT_IN2OUT_NEXT_DROP;
 733       goto out;
 734     }
 735
 736 out:
 737   *p_value = sm0;
 738 out2:
 739   *p_proto = key0.protocol;
 740   *p_dont_translate = dont_translate;
 741   return next0;
 742 }
 743
 744 static inline u32 icmp_in2out (snat_main_t *sm,
 745                                vlib_buffer_t * b0,
 746                                ip4_header_t * ip0,
 747                                icmp46_header_t * icmp0,
 748                                u32 sw_if_index0,
 749                                u32 rx_fib_index0,
 750                                vlib_node_runtime_t * node,
 751                                u32 next0,
 752                                u32 thread_index,
 753                                void *d,
 754                                void *e)
 755 {
 756   snat_session_key_t sm0;
 757   u8 protocol;
 758   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 759   ip4_header_t *inner_ip0;
 760   void *l4_header = 0;
 761   icmp46_header_t *inner_icmp0;
 762   u8 dont_translate;
 763   u32 new_addr0, old_addr0;
 764   u16 old_id0, new_id0;
 765   ip_csum_t sum0;
 766   u16 checksum0;
 767   u32 next0_tmp;
 768
 769   echo0 = (icmp_echo_header_t *)(icmp0+1);
 770
 771   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 772                                        &protocol, &sm0, &dont_translate, d, e);
 773   if (next0_tmp != ~0)
 774     next0 = next0_tmp;
 775   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 776     goto out;
 777
 778   sum0 = ip_incremental_checksum (0, icmp0,
 779                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 780   checksum0 = ~ip_csum_fold (sum0);
 781   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 782     {
 783       next0 = SNAT_IN2OUT_NEXT_DROP;
 784       goto out;
 785     }
 786
 787   old_addr0 = ip0->src_address.as_u32;
 788   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 789   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 790     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 791
 792   sum0 = ip0->checksum;
 793   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 794                          src_address /* changed member */);
 795   ip0->checksum = ip_csum_fold (sum0);
 796
 797   if (icmp0->checksum == 0)
 798     icmp0->checksum = 0xffff;
 799
 800   if (!icmp_is_error_message (icmp0))
 801     {
 802       new_id0 = sm0.port;
 803       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 804         {
 805           old_id0 = echo0->identifier;
 806           new_id0 = sm0.port;
 807           echo0->identifier = new_id0;
 808
 809           sum0 = icmp0->checksum;
 810           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 811                                  identifier);
 812           icmp0->checksum = ip_csum_fold (sum0);
 813         }
 814     }
 815   else
 816     {
 817       inner_ip0 = (ip4_header_t *)(echo0+1);
 818       l4_header = ip4_next_header (inner_ip0);
 819
 820       if (!ip4_header_checksum_is_valid (inner_ip0))
 821         {
 822           next0 = SNAT_IN2OUT_NEXT_DROP;
 823           goto out;
 824         }
 825
 826       old_addr0 = inner_ip0->dst_address.as_u32;
 827       inner_ip0->dst_address = sm0.addr;
 828       new_addr0 = inner_ip0->dst_address.as_u32;
 829
 830       sum0 = icmp0->checksum;
 831       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 832                              dst_address /* changed member */);
 833       icmp0->checksum = ip_csum_fold (sum0);
 834
 835       switch (protocol)
 836         {
 837           case SNAT_PROTOCOL_ICMP:
 838             inner_icmp0 = (icmp46_header_t*)l4_header;
 839             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 840
 841             old_id0 = inner_echo0->identifier;
 842             new_id0 = sm0.port;
 843             inner_echo0->identifier = new_id0;
 844
 845             sum0 = icmp0->checksum;
 846             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 847                                    identifier);
 848             icmp0->checksum = ip_csum_fold (sum0);
 849             break;
 850           case SNAT_PROTOCOL_UDP:
 851           case SNAT_PROTOCOL_TCP:
 852             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 853             new_id0 = sm0.port;
 854             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 855
 856             sum0 = icmp0->checksum;
 857             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 858                                    dst_port);
 859             icmp0->checksum = ip_csum_fold (sum0);
 860             break;
 861           default:
 862             ASSERT(0);
 863         }
 864     }
 865
 866 out:
 867   return next0;
 868 }
 869
 870 /**
 871  * @brief Hairpinning
 872  *
 873  * Hairpinning allows two endpoints on the internal side of the NAT to
 874  * communicate even if they only use each other's external IP addresses
 875  * and ports.
 876  *
 877  * @param sm     NAT main.
 878  * @param b0     Vlib buffer.
 879  * @param ip0    IP header.
 880  * @param udp0   UDP header.
 881  * @param tcp0   TCP header.
 882  * @param proto0 NAT protocol.
 883  */
 884 static inline int
 885 snat_hairpinning (snat_main_t *sm,
 886                   vlib_buffer_t * b0,
 887                   ip4_header_t * ip0,
 888                   udp_header_t * udp0,
 889                   tcp_header_t * tcp0,
 890                   u32 proto0)
 891 {
 892   snat_session_key_t key0, sm0;
 893   snat_session_t * s0;
 894   clib_bihash_kv_8_8_t kv0, value0;
 895   ip_csum_t sum0;
 896   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 897   u16 new_dst_port0, old_dst_port0;
 898
 899   key0.addr = ip0->dst_address;
 900   key0.port = udp0->dst_port;
 901   key0.protocol = proto0;
 902   key0.fib_index = sm->outside_fib_index;
 903   kv0.key = key0.as_u64;
 904
 905   /* Check if destination is static mappings */
 906   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
 907     {
 908       new_dst_addr0 = sm0.addr.as_u32;
 909       new_dst_port0 = sm0.port;
 910       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 911     }
 912   /* or active session */
 913   else
 914     {
 915       if (sm->num_workers > 1)
 916         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 917       else
 918         ti = sm->num_workers;
 919
 920       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 921         {
 922           si = value0.value;
 923
 924           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 925           new_dst_addr0 = s0->in2out.addr.as_u32;
 926           new_dst_port0 = s0->in2out.port;
 927           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 928         }
 929     }
 930
 931   /* Destination is behind the same NAT, use internal address and port */
 932   if (new_dst_addr0)
 933     {
 934       old_dst_addr0 = ip0->dst_address.as_u32;
 935       ip0->dst_address.as_u32 = new_dst_addr0;
 936       sum0 = ip0->checksum;
 937       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 938                              ip4_header_t, dst_address);
 939       ip0->checksum = ip_csum_fold (sum0);
 940
 941       old_dst_port0 = tcp0->dst;
 942       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 943         {
 944           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 945             {
 946               tcp0->dst = new_dst_port0;
 947               sum0 = tcp0->checksum;
 948               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 949                                      ip4_header_t, dst_address);
 950               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 951                                      ip4_header_t /* cheat */, length);
 952               tcp0->checksum = ip_csum_fold(sum0);
 953             }
 954           else
 955             {
 956               udp0->dst_port = new_dst_port0;
 957               udp0->checksum = 0;
 958             }
 959         }
 960       else
 961         {
 962           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 963             {
 964               sum0 = tcp0->checksum;
 965               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 966                                      ip4_header_t, dst_address);
 967               tcp0->checksum = ip_csum_fold(sum0);
 968             }
 969         }
 970       return 1;
 971     }
 972   return 0;
 973 }
 974
 975 static inline void
 976 snat_icmp_hairpinning (snat_main_t *sm,
 977                        vlib_buffer_t * b0,
 978                        ip4_header_t * ip0,
 979                        icmp46_header_t * icmp0)
 980 {
 981   snat_session_key_t key0, sm0;
 982   clib_bihash_kv_8_8_t kv0, value0;
 983   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 984   ip_csum_t sum0;
 985   snat_session_t *s0;
 986
 987   if (!icmp_is_error_message (icmp0))
 988     {
 989       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 990       u16 icmp_id0 = echo0->identifier;
 991       key0.addr = ip0->dst_address;
 992       key0.port = icmp_id0;
 993       key0.protocol = SNAT_PROTOCOL_ICMP;
 994       key0.fib_index = sm->outside_fib_index;
 995       kv0.key = key0.as_u64;
 996
 997       if (sm->num_workers > 1)
 998         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 999       else
1000         ti = sm->num_workers;
1001
1002       /* Check if destination is in active sessions */
1003       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1004                                   &value0))
1005         {
1006           /* or static mappings */
1007           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
1008             {
1009               new_dst_addr0 = sm0.addr.as_u32;
1010               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1011             }
1012         }
1013       else
1014         {
1015           si = value0.value;
1016
1017           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1018           new_dst_addr0 = s0->in2out.addr.as_u32;
1019           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1020           echo0->identifier = s0->in2out.port;
1021           sum0 = icmp0->checksum;
1022           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1023                                  icmp_echo_header_t, identifier);
1024           icmp0->checksum = ip_csum_fold (sum0);
1025         }
1026
1027       /* Destination is behind the same NAT, use internal address and port */
1028       if (new_dst_addr0)
1029         {
1030           old_dst_addr0 = ip0->dst_address.as_u32;
1031           ip0->dst_address.as_u32 = new_dst_addr0;
1032           sum0 = ip0->checksum;
1033           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1034                                  ip4_header_t, dst_address);
1035           ip0->checksum = ip_csum_fold (sum0);
1036         }
1037     }
1038
1039 }
1040
1041 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1042                                          vlib_buffer_t * b0,
1043                                          ip4_header_t * ip0,
1044                                          icmp46_header_t * icmp0,
1045                                          u32 sw_if_index0,
1046                                          u32 rx_fib_index0,
1047                                          vlib_node_runtime_t * node,
1048                                          u32 next0,
1049                                          f64 now,
1050                                          u32 thread_index,
1051                                          snat_session_t ** p_s0)
1052 {
1053   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1054                       next0, thread_index, p_s0, 0);
1055   snat_session_t * s0 = *p_s0;
1056   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1057     {
1058       /* Hairpinning */
1059       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1060         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1061       /* Accounting */
1062       s0->last_heard = now;
1063       s0->total_pkts++;
1064       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1065       /* Per-user LRU list maintenance */
1066       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1067                          s0->per_user_index);
1068       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1069                           s0->per_user_list_head_index,
1070                           s0->per_user_index);
1071     }
1072   return next0;
1073 }
1074 static inline void
1075 snat_hairpinning_unknown_proto (snat_main_t *sm,
1076                                 vlib_buffer_t * b,
1077                                 ip4_header_t * ip)
1078 {
1079   u32 old_addr, new_addr = 0, ti = 0;
1080   clib_bihash_kv_8_8_t kv, value;
1081   clib_bihash_kv_16_8_t s_kv, s_value;
1082   nat_ed_ses_key_t key;
1083   snat_session_key_t m_key;
1084   snat_static_mapping_t *m;
1085   ip_csum_t sum;
1086   snat_session_t *s;
1087
1088   old_addr = ip->dst_address.as_u32;
1089   key.l_addr.as_u32 = ip->dst_address.as_u32;
1090   key.r_addr.as_u32 = ip->src_address.as_u32;
1091   key.fib_index = sm->outside_fib_index;
1092   key.proto = ip->protocol;
1093   key.r_port = 0;
1094   key.l_port = 0;
1095   s_kv.key[0] = key.as_u64[0];
1096   s_kv.key[1] = key.as_u64[1];
1097   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1098     {
1099       m_key.addr = ip->dst_address;
1100       m_key.fib_index = sm->outside_fib_index;
1101       m_key.port = 0;
1102       m_key.protocol = 0;
1103       kv.key = m_key.as_u64;
1104       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1105         return;
1106
1107       m = pool_elt_at_index (sm->static_mappings, value.value);
1108       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1109         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1110       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1111     }
1112   else
1113     {
1114       if (sm->num_workers > 1)
1115         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1116       else
1117         ti = sm->num_workers;
1118
1119       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1120       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1121         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1122       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1123     }
1124   sum = ip->checksum;
1125   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1126   ip->checksum = ip_csum_fold (sum);
1127 }
1128
1129 static snat_session_t *
1130 snat_in2out_unknown_proto (snat_main_t *sm,
1131                            vlib_buffer_t * b,
1132                            ip4_header_t * ip,
1133                            u32 rx_fib_index,
1134                            u32 thread_index,
1135                            f64 now,
1136                            vlib_main_t * vm,
1137                            vlib_node_runtime_t * node)
1138 {
1139   clib_bihash_kv_8_8_t kv, value;
1140   clib_bihash_kv_16_8_t s_kv, s_value;
1141   snat_static_mapping_t *m;
1142   snat_session_key_t m_key;
1143   u32 old_addr, new_addr = 0;
1144   ip_csum_t sum;
1145   snat_user_t *u;
1146   dlist_elt_t *head, *elt;
1147   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1148   u32 elt_index, head_index, ses_index;
1149   snat_session_t * s;
1150   nat_ed_ses_key_t key;
1151   u32 address_index = ~0;
1152   int i;
1153   u8 is_sm = 0;
1154
1155   old_addr = ip->src_address.as_u32;
1156
1157   key.l_addr = ip->src_address;
1158   key.r_addr = ip->dst_address;
1159   key.fib_index = rx_fib_index;
1160   key.proto = ip->protocol;
1161   key.l_port = 0;
1162   key.r_port = 0;
1163   s_kv.key[0] = key.as_u64[0];
1164   s_kv.key[1] = key.as_u64[1];
1165
1166   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1167     {
1168       s = pool_elt_at_index (tsm->sessions, s_value.value);
1169       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1170     }
1171   else
1172     {
1173       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1174         {
1175           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1176           nat_ipfix_logging_max_sessions(sm->max_translations);
1177           return 0;
1178         }
1179
1180       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1181                                   thread_index);
1182       if (!u)
1183         {
1184           clib_warning ("create NAT user failed");
1185           return 0;
1186         }
1187
1188       m_key.addr = ip->src_address;
1189       m_key.port = 0;
1190       m_key.protocol = 0;
1191       m_key.fib_index = rx_fib_index;
1192       kv.key = m_key.as_u64;
1193
1194       /* Try to find static mapping first */
1195       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1196         {
1197           m = pool_elt_at_index (sm->static_mappings, value.value);
1198           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1199           is_sm = 1;
1200           goto create_ses;
1201         }
1202       /* Fallback to 3-tuple key */
1203       else
1204         {
1205           /* Choose same out address as for TCP/UDP session to same destination */
1206           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1207             {
1208               head_index = u->sessions_per_user_list_head_index;
1209               head = pool_elt_at_index (tsm->list_pool, head_index);
1210               elt_index = head->next;
1211               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1212               ses_index = elt->value;
1213               while (ses_index != ~0)
1214                 {
1215                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1216                   elt_index = elt->next;
1217                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1218                   ses_index = elt->value;
1219
1220                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1221                     {
1222                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1223                       address_index = s->outside_address_index;
1224
1225                       key.fib_index = sm->outside_fib_index;
1226                       key.l_addr.as_u32 = new_addr;
1227                       s_kv.key[0] = key.as_u64[0];
1228                       s_kv.key[1] = key.as_u64[1];
1229                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1230                         break;
1231
1232                       goto create_ses;
1233                     }
1234                 }
1235             }
1236           key.fib_index = sm->outside_fib_index;
1237           for (i = 0; i < vec_len (sm->addresses); i++)
1238             {
1239               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1240               s_kv.key[0] = key.as_u64[0];
1241               s_kv.key[1] = key.as_u64[1];
1242               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1243                 {
1244                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1245                   address_index = i;
1246                   goto create_ses;
1247                 }
1248             }
1249           return 0;
1250         }
1251
1252 create_ses:
1253       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1254       if (!s)
1255         {
1256           clib_warning ("create NAT session failed");
1257           return 0;
1258         }
1259
1260       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1261       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1262       s->outside_address_index = address_index;
1263       s->out2in.addr.as_u32 = new_addr;
1264       s->out2in.fib_index = sm->outside_fib_index;
1265       s->in2out.addr.as_u32 = old_addr;
1266       s->in2out.fib_index = rx_fib_index;
1267       s->in2out.port = s->out2in.port = ip->protocol;
1268       if (is_sm)
1269         {
1270           u->nstaticsessions++;
1271           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1272         }
1273       else
1274         {
1275           u->nsessions++;
1276         }
1277
1278       /* Add to lookup tables */
1279       key.l_addr.as_u32 = old_addr;
1280       key.r_addr = ip->dst_address;
1281       key.proto = ip->protocol;
1282       key.fib_index = rx_fib_index;
1283       s_kv.key[0] = key.as_u64[0];
1284       s_kv.key[1] = key.as_u64[1];
1285       s_kv.value = s - tsm->sessions;
1286       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1287         clib_warning ("in2out key add failed");
1288
1289       key.l_addr.as_u32 = new_addr;
1290       key.fib_index = sm->outside_fib_index;
1291       s_kv.key[0] = key.as_u64[0];
1292       s_kv.key[1] = key.as_u64[1];
1293       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1294         clib_warning ("out2in key add failed");
1295   }
1296
1297   /* Update IP checksum */
1298   sum = ip->checksum;
1299   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1300   ip->checksum = ip_csum_fold (sum);
1301
1302   /* Accounting */
1303   s->last_heard = now;
1304   s->total_pkts++;
1305   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1306   /* Per-user LRU list maintenance */
1307   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1308   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1309                       s->per_user_index);
1310
1311   /* Hairpinning */
1312   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1313     snat_hairpinning_unknown_proto(sm, b, ip);
1314
1315   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1316     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1317
1318   return s;
1319 }
1320
1321 static snat_session_t *
1322 snat_in2out_lb (snat_main_t *sm,
1323                 vlib_buffer_t * b,
1324                 ip4_header_t * ip,
1325                 u32 rx_fib_index,
1326                 u32 thread_index,
1327                 f64 now,
1328                 vlib_main_t * vm,
1329                 vlib_node_runtime_t * node)
1330 {
1331   nat_ed_ses_key_t key;
1332   clib_bihash_kv_16_8_t s_kv, s_value;
1333   udp_header_t *udp = ip4_next_header (ip);
1334   tcp_header_t *tcp = (tcp_header_t *) udp;
1335   snat_session_t *s = 0;
1336   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1337   u32 old_addr, new_addr;
1338   u16 new_port, old_port;
1339   ip_csum_t sum;
1340   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1341   snat_session_key_t e_key, l_key;
1342   snat_user_t *u;
1343
1344   old_addr = ip->src_address.as_u32;
1345
1346   key.l_addr = ip->src_address;
1347   key.r_addr = ip->dst_address;
1348   key.fib_index = rx_fib_index;
1349   key.proto = ip->protocol;
1350   key.r_port = udp->dst_port;
1351   key.l_port = udp->src_port;
1352   s_kv.key[0] = key.as_u64[0];
1353   s_kv.key[1] = key.as_u64[1];
1354
1355   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1356     {
1357       if (s_value.value == ~0ULL)
1358         return 0;
1359       s = pool_elt_at_index (tsm->sessions, s_value.value);
1360     }
1361   else
1362     {
1363       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1364         {
1365           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1366           nat_ipfix_logging_max_sessions(sm->max_translations);
1367           return 0;
1368         }
1369
1370       l_key.addr = ip->src_address;
1371       l_key.port = udp->src_port;
1372       l_key.protocol = proto;
1373       l_key.fib_index = rx_fib_index;
1374       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0))
1375         return 0;
1376
1377       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1378                                   thread_index);
1379       if (!u)
1380         {
1381           clib_warning ("create NAT user failed");
1382           return 0;
1383         }
1384
1385       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1386       if (!s)
1387         {
1388           clib_warning ("create NAT session failed");
1389           return 0;
1390         }
1391
1392       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1393       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1394       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1395       s->outside_address_index = ~0;
1396       s->in2out = l_key;
1397       s->out2in = e_key;
1398       s->out2in.protocol = l_key.protocol;
1399       u->nstaticsessions++;
1400
1401       /* Add to lookup tables */
1402       s_kv.value = s - tsm->sessions;
1403       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1404         clib_warning ("in2out-ed key add failed");
1405
1406       key.l_addr = e_key.addr;
1407       key.fib_index = e_key.fib_index;
1408       key.l_port = e_key.port;
1409       s_kv.key[0] = key.as_u64[0];
1410       s_kv.key[1] = key.as_u64[1];
1411       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1412         clib_warning ("out2in-ed key add failed");
1413     }
1414
1415   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1416
1417   /* Update IP checksum */
1418   sum = ip->checksum;
1419   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1420   if (is_twice_nat_session (s))
1421     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1422                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1423   ip->checksum = ip_csum_fold (sum);
1424
1425   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1426     {
1427       old_port = tcp->src_port;
1428       tcp->src_port = s->out2in.port;
1429       new_port = tcp->src_port;
1430
1431       sum = tcp->checksum;
1432       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1433       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1434       if (is_twice_nat_session (s))
1435         {
1436           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1437                                 s->ext_host_addr.as_u32, ip4_header_t,
1438                                 dst_address);
1439           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1440                                 ip4_header_t, length);
1441           tcp->dst_port = s->ext_host_port;
1442           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1443         }
1444       tcp->checksum = ip_csum_fold(sum);
1445     }
1446   else
1447     {
1448       udp->src_port = s->out2in.port;
1449       if (is_twice_nat_session (s))
1450         {
1451           udp->dst_port = s->ext_host_port;
1452           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1453         }
1454       udp->checksum = 0;
1455     }
1456
1457   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1458     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1459
1460   /* Accounting */
1461   s->last_heard = now;
1462   s->total_pkts++;
1463   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1464   /* Per-user LRU list maintenance */
1465   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1466   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1467                       s->per_user_index);
1468   return s;
1469 }
1470
1471 static inline uword
1472 snat_in2out_node_fn_inline (vlib_main_t * vm,
1473                             vlib_node_runtime_t * node,
1474                             vlib_frame_t * frame, int is_slow_path,
1475                             int is_output_feature)
1476 {
1477   u32 n_left_from, * from, * to_next;
1478   snat_in2out_next_t next_index;
1479   u32 pkts_processed = 0;
1480   snat_main_t * sm = &snat_main;
1481   f64 now = vlib_time_now (vm);
1482   u32 stats_node_index;
1483   u32 thread_index = vlib_get_thread_index ();
1484
1485   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1486     snat_in2out_node.index;
1487
1488   from = vlib_frame_vector_args (frame);
1489   n_left_from = frame->n_vectors;
1490   next_index = node->cached_next_index;
1491
1492   while (n_left_from > 0)
1493     {
1494       u32 n_left_to_next;
1495
1496       vlib_get_next_frame (vm, node, next_index,
1497                            to_next, n_left_to_next);
1498
1499       while (n_left_from >= 4 && n_left_to_next >= 2)
1500         {
1501           u32 bi0, bi1;
1502           vlib_buffer_t * b0, * b1;
1503           u32 next0, next1;
1504           u32 sw_if_index0, sw_if_index1;
1505           ip4_header_t * ip0, * ip1;
1506           ip_csum_t sum0, sum1;
1507           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1508           u16 old_port0, new_port0, old_port1, new_port1;
1509           udp_header_t * udp0, * udp1;
1510           tcp_header_t * tcp0, * tcp1;
1511           icmp46_header_t * icmp0, * icmp1;
1512           snat_session_key_t key0, key1;
1513           u32 rx_fib_index0, rx_fib_index1;
1514           u32 proto0, proto1;
1515           snat_session_t * s0 = 0, * s1 = 0;
1516           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1517           u32 iph_offset0 = 0, iph_offset1 = 0;
1518
1519           /* Prefetch next iteration. */
1520           {
1521             vlib_buffer_t * p2, * p3;
1522
1523             p2 = vlib_get_buffer (vm, from[2]);
1524             p3 = vlib_get_buffer (vm, from[3]);
1525
1526             vlib_prefetch_buffer_header (p2, LOAD);
1527             vlib_prefetch_buffer_header (p3, LOAD);
1528
1529             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1530             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1531           }
1532
1533           /* speculatively enqueue b0 and b1 to the current next frame */
1534           to_next[0] = bi0 = from[0];
1535           to_next[1] = bi1 = from[1];
1536           from += 2;
1537           to_next += 2;
1538           n_left_from -= 2;
1539           n_left_to_next -= 2;
1540
1541           b0 = vlib_get_buffer (vm, bi0);
1542           b1 = vlib_get_buffer (vm, bi1);
1543
1544           if (is_output_feature)
1545             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1546
1547           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1548                  iph_offset0);
1549
1550           udp0 = ip4_next_header (ip0);
1551           tcp0 = (tcp_header_t *) udp0;
1552           icmp0 = (icmp46_header_t *) udp0;
1553
1554           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1555           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1556                                    sw_if_index0);
1557
1558           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1559
1560           if (PREDICT_FALSE(ip0->ttl == 1))
1561             {
1562               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1563               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1564                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1565                                            0);
1566               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1567               goto trace00;
1568             }
1569
1570           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1571
1572           /* Next configured feature, probably ip4-lookup */
1573           if (is_slow_path)
1574             {
1575               if (PREDICT_FALSE (proto0 == ~0))
1576                 {
1577                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1578                                                   thread_index, now, vm, node);
1579                   if (!s0)
1580                     next0 = SNAT_IN2OUT_NEXT_DROP;
1581                   goto trace00;
1582                 }
1583
1584               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1585                 {
1586                   next0 = icmp_in2out_slow_path
1587                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1588                      node, next0, now, thread_index, &s0);
1589                   goto trace00;
1590                 }
1591             }
1592           else
1593             {
1594               if (is_output_feature)
1595                 {
1596                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0)))
1597                     goto trace00;
1598                 }
1599
1600               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1601                 {
1602                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1603                   goto trace00;
1604                 }
1605
1606               if (ip4_is_fragment (ip0))
1607                 {
1608                   next0 = SNAT_IN2OUT_NEXT_REASS;
1609                   goto trace00;
1610                 }
1611             }
1612
1613           key0.addr = ip0->src_address;
1614           key0.port = udp0->src_port;
1615           key0.protocol = proto0;
1616           key0.fib_index = rx_fib_index0;
1617
1618           kv0.key = key0.as_u64;
1619
1620           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1621               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1622             {
1623               if (is_slow_path)
1624                 {
1625                   if (is_output_feature)
1626                     {
1627                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1628                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1629                         goto trace00;
1630                     }
1631                   else
1632                     {
1633                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1634                           ip0, proto0, rx_fib_index0, thread_index)))
1635                         goto trace00;
1636                     }
1637
1638                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1639                                      &s0, node, next0, thread_index);
1640                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1641                     goto trace00;
1642                 }
1643               else
1644                 {
1645                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1646                   goto trace00;
1647                 }
1648             }
1649           else
1650             {
1651               if (PREDICT_FALSE (value0.value == ~0ULL))
1652                 {
1653                   if (is_slow_path)
1654                     {
1655                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1656                                           thread_index, now, vm, node);
1657                       if (!s0 && !sm->forwarding_enabled)
1658                         next0 = SNAT_IN2OUT_NEXT_DROP;
1659                       goto trace00;
1660                     }
1661                   else
1662                     {
1663                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1664                       goto trace00;
1665                     }
1666                 }
1667               else
1668                 {
1669                   s0 = pool_elt_at_index (
1670                     sm->per_thread_data[thread_index].sessions,
1671                     value0.value);
1672                 }
1673             }
1674
1675           b0->flags |= VNET_BUFFER_F_IS_NATED;
1676
1677           old_addr0 = ip0->src_address.as_u32;
1678           ip0->src_address = s0->out2in.addr;
1679           new_addr0 = ip0->src_address.as_u32;
1680           if (!is_output_feature)
1681             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1682
1683           sum0 = ip0->checksum;
1684           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1685                                  ip4_header_t,
1686                                  src_address /* changed member */);
1687           ip0->checksum = ip_csum_fold (sum0);
1688
1689           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1690             {
1691               old_port0 = tcp0->src_port;
1692               tcp0->src_port = s0->out2in.port;
1693               new_port0 = tcp0->src_port;
1694
1695               sum0 = tcp0->checksum;
1696               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1697                                      ip4_header_t,
1698                                      dst_address /* changed member */);
1699               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1700                                      ip4_header_t /* cheat */,
1701                                      length /* changed member */);
1702               tcp0->checksum = ip_csum_fold(sum0);
1703             }
1704           else
1705             {
1706               old_port0 = udp0->src_port;
1707               udp0->src_port = s0->out2in.port;
1708               udp0->checksum = 0;
1709             }
1710
1711           /* Accounting */
1712           s0->last_heard = now;
1713           s0->total_pkts++;
1714           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1715           /* Per-user LRU list maintenance */
1716           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1717                              s0->per_user_index);
1718           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1719                               s0->per_user_list_head_index,
1720                               s0->per_user_index);
1721         trace00:
1722
1723           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1724                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1725             {
1726               snat_in2out_trace_t *t =
1727                  vlib_add_trace (vm, node, b0, sizeof (*t));
1728               t->is_slow_path = is_slow_path;
1729               t->sw_if_index = sw_if_index0;
1730               t->next_index = next0;
1731                   t->session_index = ~0;
1732               if (s0)
1733                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1734             }
1735
1736           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1737
1738           if (is_output_feature)
1739             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1740
1741           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1742                  iph_offset1);
1743
1744           udp1 = ip4_next_header (ip1);
1745           tcp1 = (tcp_header_t *) udp1;
1746           icmp1 = (icmp46_header_t *) udp1;
1747
1748           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1749           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1750                                    sw_if_index1);
1751
1752           if (PREDICT_FALSE(ip1->ttl == 1))
1753             {
1754               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1755               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1756                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1757                                            0);
1758               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1759               goto trace01;
1760             }
1761
1762           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1763
1764           /* Next configured feature, probably ip4-lookup */
1765           if (is_slow_path)
1766             {
1767               if (PREDICT_FALSE (proto1 == ~0))
1768                 {
1769                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1770                                                   thread_index, now, vm, node);
1771                   if (!s1)
1772                     next1 = SNAT_IN2OUT_NEXT_DROP;
1773                   goto trace01;
1774                 }
1775
1776               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1777                 {
1778                   next1 = icmp_in2out_slow_path
1779                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1780                      next1, now, thread_index, &s1);
1781                   goto trace01;
1782                 }
1783             }
1784           else
1785             {
1786               if (is_output_feature)
1787                 {
1788                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip1)))
1789                     goto trace01;
1790                 }
1791
1792               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1793                 {
1794                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1795                   goto trace01;
1796                 }
1797
1798               if (ip4_is_fragment (ip1))
1799                 {
1800                   next1 = SNAT_IN2OUT_NEXT_REASS;
1801                   goto trace01;
1802                 }
1803             }
1804
1805           key1.addr = ip1->src_address;
1806           key1.port = udp1->src_port;
1807           key1.protocol = proto1;
1808           key1.fib_index = rx_fib_index1;
1809
1810           kv1.key = key1.as_u64;
1811
1812             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1813                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1814             {
1815               if (is_slow_path)
1816                 {
1817                   if (is_output_feature)
1818                     {
1819                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1820                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1821                         goto trace01;
1822                     }
1823                   else
1824                     {
1825                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1826                           ip1, proto1, rx_fib_index1, thread_index)))
1827                         goto trace01;
1828                     }
1829
1830                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1831                                      &s1, node, next1, thread_index);
1832                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1833                     goto trace01;
1834                 }
1835               else
1836                 {
1837                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1838                   goto trace01;
1839                 }
1840             }
1841           else
1842             {
1843               if (PREDICT_FALSE (value1.value == ~0ULL))
1844                 {
1845                   if (is_slow_path)
1846                     {
1847                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1848                                           thread_index, now, vm, node);
1849                       if (!s1 && !sm->forwarding_enabled)
1850                         next1 = SNAT_IN2OUT_NEXT_DROP;
1851                       goto trace01;
1852                     }
1853                   else
1854                     {
1855                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1856                       goto trace01;
1857                     }
1858                 }
1859               else
1860                 {
1861                   s1 = pool_elt_at_index (
1862                     sm->per_thread_data[thread_index].sessions,
1863                     value1.value);
1864                 }
1865             }
1866
1867           b1->flags |= VNET_BUFFER_F_IS_NATED;
1868
1869           old_addr1 = ip1->src_address.as_u32;
1870           ip1->src_address = s1->out2in.addr;
1871           new_addr1 = ip1->src_address.as_u32;
1872           if (!is_output_feature)
1873             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1874
1875           sum1 = ip1->checksum;
1876           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1877                                  ip4_header_t,
1878                                  src_address /* changed member */);
1879           ip1->checksum = ip_csum_fold (sum1);
1880
1881           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1882             {
1883               old_port1 = tcp1->src_port;
1884               tcp1->src_port = s1->out2in.port;
1885               new_port1 = tcp1->src_port;
1886
1887               sum1 = tcp1->checksum;
1888               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1889                                      ip4_header_t,
1890                                      dst_address /* changed member */);
1891               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1892                                      ip4_header_t /* cheat */,
1893                                      length /* changed member */);
1894               tcp1->checksum = ip_csum_fold(sum1);
1895             }
1896           else
1897             {
1898               old_port1 = udp1->src_port;
1899               udp1->src_port = s1->out2in.port;
1900               udp1->checksum = 0;
1901             }
1902
1903           /* Accounting */
1904           s1->last_heard = now;
1905           s1->total_pkts++;
1906           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1907           /* Per-user LRU list maintenance */
1908           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1909                              s1->per_user_index);
1910           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1911                               s1->per_user_list_head_index,
1912                               s1->per_user_index);
1913         trace01:
1914
1915           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1916                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1917             {
1918               snat_in2out_trace_t *t =
1919                  vlib_add_trace (vm, node, b1, sizeof (*t));
1920               t->sw_if_index = sw_if_index1;
1921               t->next_index = next1;
1922               t->session_index = ~0;
1923               if (s1)
1924                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1925             }
1926
1927           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1928
1929           /* verify speculative enqueues, maybe switch current next frame */
1930           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1931                                            to_next, n_left_to_next,
1932                                            bi0, bi1, next0, next1);
1933         }
1934
1935       while (n_left_from > 0 && n_left_to_next > 0)
1936         {
1937           u32 bi0;
1938           vlib_buffer_t * b0;
1939           u32 next0;
1940           u32 sw_if_index0;
1941           ip4_header_t * ip0;
1942           ip_csum_t sum0;
1943           u32 new_addr0, old_addr0;
1944           u16 old_port0, new_port0;
1945           udp_header_t * udp0;
1946           tcp_header_t * tcp0;
1947           icmp46_header_t * icmp0;
1948           snat_session_key_t key0;
1949           u32 rx_fib_index0;
1950           u32 proto0;
1951           snat_session_t * s0 = 0;
1952           clib_bihash_kv_8_8_t kv0, value0;
1953           u32 iph_offset0 = 0;
1954
1955           /* speculatively enqueue b0 to the current next frame */
1956           bi0 = from[0];
1957           to_next[0] = bi0;
1958           from += 1;
1959           to_next += 1;
1960           n_left_from -= 1;
1961           n_left_to_next -= 1;
1962
1963           b0 = vlib_get_buffer (vm, bi0);
1964           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1965
1966           if (is_output_feature)
1967             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1968
1969           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1970                  iph_offset0);
1971
1972           udp0 = ip4_next_header (ip0);
1973           tcp0 = (tcp_header_t *) udp0;
1974           icmp0 = (icmp46_header_t *) udp0;
1975
1976           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1977           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1978                                    sw_if_index0);
1979
1980           if (PREDICT_FALSE(ip0->ttl == 1))
1981             {
1982               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1983               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1984                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1985                                            0);
1986               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1987               goto trace0;
1988             }
1989
1990           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1991
1992           /* Next configured feature, probably ip4-lookup */
1993           if (is_slow_path)
1994             {
1995               if (PREDICT_FALSE (proto0 == ~0))
1996                 {
1997                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1998                                                   thread_index, now, vm, node);
1999                   if (!s0)
2000                     next0 = SNAT_IN2OUT_NEXT_DROP;
2001                   goto trace0;
2002                 }
2003
2004               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2005                 {
2006                   next0 = icmp_in2out_slow_path
2007                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2008                      next0, now, thread_index, &s0);
2009                   goto trace0;
2010                 }
2011             }
2012           else
2013             {
2014                if (is_output_feature)
2015                 {
2016                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0)))
2017                     goto trace0;
2018                 }
2019
2020               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2021                 {
2022                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2023                   goto trace0;
2024                 }
2025
2026               if (ip4_is_fragment (ip0))
2027                 {
2028                   next0 = SNAT_IN2OUT_NEXT_REASS;
2029                   goto trace0;
2030                 }
2031             }
2032
2033           key0.addr = ip0->src_address;
2034           key0.port = udp0->src_port;
2035           key0.protocol = proto0;
2036           key0.fib_index = rx_fib_index0;
2037
2038           kv0.key = key0.as_u64;
2039
2040           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2041                                       &kv0, &value0))
2042             {
2043               if (is_slow_path)
2044                 {
2045                   if (is_output_feature)
2046                     {
2047                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
2048                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
2049                         goto trace0;
2050                     }
2051                   else
2052                     {
2053                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2054                           ip0, proto0, rx_fib_index0, thread_index)))
2055                         goto trace0;
2056                     }
2057
2058                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2059                                      &s0, node, next0, thread_index);
2060
2061                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2062                     goto trace0;
2063                 }
2064               else
2065                 {
2066                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2067                   goto trace0;
2068                 }
2069             }
2070           else
2071             {
2072               if (PREDICT_FALSE (value0.value == ~0ULL))
2073                 {
2074                   if (is_slow_path)
2075                     {
2076                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2077                                           thread_index, now, vm, node);
2078                       if (!s0 && !sm->forwarding_enabled)
2079                         next0 = SNAT_IN2OUT_NEXT_DROP;
2080                       goto trace0;
2081                     }
2082                   else
2083                     {
2084                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2085                       goto trace0;
2086                     }
2087                 }
2088               else
2089                 {
2090                   s0 = pool_elt_at_index (
2091                     sm->per_thread_data[thread_index].sessions,
2092                     value0.value);
2093                 }
2094             }
2095
2096           b0->flags |= VNET_BUFFER_F_IS_NATED;
2097
2098           old_addr0 = ip0->src_address.as_u32;
2099           ip0->src_address = s0->out2in.addr;
2100           new_addr0 = ip0->src_address.as_u32;
2101           if (!is_output_feature)
2102             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2103
2104           sum0 = ip0->checksum;
2105           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2106                                  ip4_header_t,
2107                                  src_address /* changed member */);
2108           ip0->checksum = ip_csum_fold (sum0);
2109
2110           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2111             {
2112               old_port0 = tcp0->src_port;
2113               tcp0->src_port = s0->out2in.port;
2114               new_port0 = tcp0->src_port;
2115
2116               sum0 = tcp0->checksum;
2117               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2118                                      ip4_header_t,
2119                                      dst_address /* changed member */);
2120               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2121                                      ip4_header_t /* cheat */,
2122                                      length /* changed member */);
2123               tcp0->checksum = ip_csum_fold(sum0);
2124             }
2125           else
2126             {
2127               old_port0 = udp0->src_port;
2128               udp0->src_port = s0->out2in.port;
2129               udp0->checksum = 0;
2130             }
2131
2132           /* Accounting */
2133           s0->last_heard = now;
2134           s0->total_pkts++;
2135           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2136           /* Per-user LRU list maintenance */
2137           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2138                              s0->per_user_index);
2139           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2140                               s0->per_user_list_head_index,
2141                               s0->per_user_index);
2142
2143         trace0:
2144           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2145                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2146             {
2147               snat_in2out_trace_t *t =
2148                  vlib_add_trace (vm, node, b0, sizeof (*t));
2149               t->is_slow_path = is_slow_path;
2150               t->sw_if_index = sw_if_index0;
2151               t->next_index = next0;
2152                   t->session_index = ~0;
2153               if (s0)
2154                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2155             }
2156
2157           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2158
2159           /* verify speculative enqueue, maybe switch current next frame */
2160           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2161                                            to_next, n_left_to_next,
2162                                            bi0, next0);
2163         }
2164
2165       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2166     }
2167
2168   vlib_node_increment_counter (vm, stats_node_index,
2169                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2170                                pkts_processed);
2171   return frame->n_vectors;
2172 }
2173
2174 static uword
2175 snat_in2out_fast_path_fn (vlib_main_t * vm,
2176                           vlib_node_runtime_t * node,
2177                           vlib_frame_t * frame)
2178 {
2179   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2180 }
2181
2182 VLIB_REGISTER_NODE (snat_in2out_node) = {
2183   .function = snat_in2out_fast_path_fn,
2184   .name = "nat44-in2out",
2185   .vector_size = sizeof (u32),
2186   .format_trace = format_snat_in2out_trace,
2187   .type = VLIB_NODE_TYPE_INTERNAL,
2188
2189   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2190   .error_strings = snat_in2out_error_strings,
2191
2192   .runtime_data_bytes = sizeof (snat_runtime_t),
2193
2194   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2195
2196   /* edit / add dispositions here */
2197   .next_nodes = {
2198     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2199     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2200     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2201     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2202     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2203   },
2204 };
2205
2206 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2207
2208 static uword
2209 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2210                                  vlib_node_runtime_t * node,
2211                                  vlib_frame_t * frame)
2212 {
2213   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2214 }
2215
2216 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2217   .function = snat_in2out_output_fast_path_fn,
2218   .name = "nat44-in2out-output",
2219   .vector_size = sizeof (u32),
2220   .format_trace = format_snat_in2out_trace,
2221   .type = VLIB_NODE_TYPE_INTERNAL,
2222
2223   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2224   .error_strings = snat_in2out_error_strings,
2225
2226   .runtime_data_bytes = sizeof (snat_runtime_t),
2227
2228   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2229
2230   /* edit / add dispositions here */
2231   .next_nodes = {
2232     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2233     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2234     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2235     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2236     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2237   },
2238 };
2239
2240 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2241                               snat_in2out_output_fast_path_fn);
2242
2243 static uword
2244 snat_in2out_slow_path_fn (vlib_main_t * vm,
2245                           vlib_node_runtime_t * node,
2246                           vlib_frame_t * frame)
2247 {
2248   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2249 }
2250
2251 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2252   .function = snat_in2out_slow_path_fn,
2253   .name = "nat44-in2out-slowpath",
2254   .vector_size = sizeof (u32),
2255   .format_trace = format_snat_in2out_trace,
2256   .type = VLIB_NODE_TYPE_INTERNAL,
2257
2258   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2259   .error_strings = snat_in2out_error_strings,
2260
2261   .runtime_data_bytes = sizeof (snat_runtime_t),
2262
2263   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2264
2265   /* edit / add dispositions here */
2266   .next_nodes = {
2267     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2268     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2269     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2270     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2271     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2272   },
2273 };
2274
2275 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2276                               snat_in2out_slow_path_fn);
2277
2278 static uword
2279 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2280                                  vlib_node_runtime_t * node,
2281                                  vlib_frame_t * frame)
2282 {
2283   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2284 }
2285
2286 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2287   .function = snat_in2out_output_slow_path_fn,
2288   .name = "nat44-in2out-output-slowpath",
2289   .vector_size = sizeof (u32),
2290   .format_trace = format_snat_in2out_trace,
2291   .type = VLIB_NODE_TYPE_INTERNAL,
2292
2293   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2294   .error_strings = snat_in2out_error_strings,
2295
2296   .runtime_data_bytes = sizeof (snat_runtime_t),
2297
2298   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2299
2300   /* edit / add dispositions here */
2301   .next_nodes = {
2302     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2303     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2304     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2305     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2306     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2307   },
2308 };
2309
2310 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2311                               snat_in2out_output_slow_path_fn);
2312
2313 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2314
2315 static uword
2316 nat44_hairpinning_fn (vlib_main_t * vm,
2317                       vlib_node_runtime_t * node,
2318                       vlib_frame_t * frame)
2319 {
2320   u32 n_left_from, * from, * to_next;
2321   snat_in2out_next_t next_index;
2322   u32 pkts_processed = 0;
2323   snat_main_t * sm = &snat_main;
2324   vnet_feature_main_t *fm = &feature_main;
2325   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2326   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2327
2328   from = vlib_frame_vector_args (frame);
2329   n_left_from = frame->n_vectors;
2330   next_index = node->cached_next_index;
2331
2332   while (n_left_from > 0)
2333     {
2334       u32 n_left_to_next;
2335
2336       vlib_get_next_frame (vm, node, next_index,
2337                            to_next, n_left_to_next);
2338
2339       while (n_left_from > 0 && n_left_to_next > 0)
2340         {
2341           u32 bi0;
2342           vlib_buffer_t * b0;
2343           u32 next0;
2344           ip4_header_t * ip0;
2345           u32 proto0;
2346           udp_header_t * udp0;
2347           tcp_header_t * tcp0;
2348
2349           /* speculatively enqueue b0 to the current next frame */
2350           bi0 = from[0];
2351           to_next[0] = bi0;
2352           from += 1;
2353           to_next += 1;
2354           n_left_from -= 1;
2355           n_left_to_next -= 1;
2356
2357           b0 = vlib_get_buffer (vm, bi0);
2358           ip0 = vlib_buffer_get_current (b0);
2359           udp0 = ip4_next_header (ip0);
2360           tcp0 = (tcp_header_t *) udp0;
2361
2362           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2363
2364           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2365                                 &next0, 0);
2366
2367           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2368             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2369
2370           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2371
2372           /* verify speculative enqueue, maybe switch current next frame */
2373           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2374                                            to_next, n_left_to_next,
2375                                            bi0, next0);
2376          }
2377
2378       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2379     }
2380
2381   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2382                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2383                                pkts_processed);
2384   return frame->n_vectors;
2385 }
2386
2387 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2388   .function = nat44_hairpinning_fn,
2389   .name = "nat44-hairpinning",
2390   .vector_size = sizeof (u32),
2391   .type = VLIB_NODE_TYPE_INTERNAL,
2392   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2393   .error_strings = snat_in2out_error_strings,
2394   .n_next_nodes = 2,
2395   .next_nodes = {
2396     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2397     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2398   },
2399 };
2400
2401 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2402                               nat44_hairpinning_fn);
2403
2404 static inline void
2405 nat44_reass_hairpinning (snat_main_t *sm,
2406                          vlib_buffer_t * b0,
2407                          ip4_header_t * ip0,
2408                          u16 sport,
2409                          u16 dport,
2410                          u32 proto0)
2411 {
2412   snat_session_key_t key0, sm0;
2413   snat_session_t * s0;
2414   clib_bihash_kv_8_8_t kv0, value0;
2415   ip_csum_t sum0;
2416   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2417   u16 new_dst_port0, old_dst_port0;
2418   udp_header_t * udp0;
2419   tcp_header_t * tcp0;
2420
2421   key0.addr = ip0->dst_address;
2422   key0.port = dport;
2423   key0.protocol = proto0;
2424   key0.fib_index = sm->outside_fib_index;
2425   kv0.key = key0.as_u64;
2426
2427   udp0 = ip4_next_header (ip0);
2428
2429   /* Check if destination is static mappings */
2430   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
2431     {
2432       new_dst_addr0 = sm0.addr.as_u32;
2433       new_dst_port0 = sm0.port;
2434       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2435     }
2436   /* or active sessions */
2437   else
2438     {
2439       if (sm->num_workers > 1)
2440         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2441       else
2442         ti = sm->num_workers;
2443
2444       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2445         {
2446           si = value0.value;
2447           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2448           new_dst_addr0 = s0->in2out.addr.as_u32;
2449           new_dst_port0 = s0->in2out.port;
2450           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2451         }
2452     }
2453
2454   /* Destination is behind the same NAT, use internal address and port */
2455   if (new_dst_addr0)
2456     {
2457       old_dst_addr0 = ip0->dst_address.as_u32;
2458       ip0->dst_address.as_u32 = new_dst_addr0;
2459       sum0 = ip0->checksum;
2460       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2461                              ip4_header_t, dst_address);
2462       ip0->checksum = ip_csum_fold (sum0);
2463
2464       old_dst_port0 = dport;
2465       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2466                        ip4_is_first_fragment (ip0)))
2467         {
2468           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2469             {
2470               tcp0 = ip4_next_header (ip0);
2471               tcp0->dst = new_dst_port0;
2472               sum0 = tcp0->checksum;
2473               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2474                                      ip4_header_t, dst_address);
2475               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2476                                      ip4_header_t /* cheat */, length);
2477               tcp0->checksum = ip_csum_fold(sum0);
2478             }
2479           else
2480             {
2481               udp0->dst_port = new_dst_port0;
2482               udp0->checksum = 0;
2483             }
2484         }
2485       else
2486         {
2487           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2488             {
2489               tcp0 = ip4_next_header (ip0);
2490               sum0 = tcp0->checksum;
2491               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2492                                      ip4_header_t, dst_address);
2493               tcp0->checksum = ip_csum_fold(sum0);
2494             }
2495         }
2496     }
2497 }
2498
2499 static uword
2500 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2501                             vlib_node_runtime_t * node,
2502                             vlib_frame_t * frame)
2503 {
2504   u32 n_left_from, *from, *to_next;
2505   snat_in2out_next_t next_index;
2506   u32 pkts_processed = 0;
2507   snat_main_t *sm = &snat_main;
2508   f64 now = vlib_time_now (vm);
2509   u32 thread_index = vlib_get_thread_index ();
2510   snat_main_per_thread_data_t *per_thread_data =
2511     &sm->per_thread_data[thread_index];
2512   u32 *fragments_to_drop = 0;
2513   u32 *fragments_to_loopback = 0;
2514
2515   from = vlib_frame_vector_args (frame);
2516   n_left_from = frame->n_vectors;
2517   next_index = node->cached_next_index;
2518
2519   while (n_left_from > 0)
2520     {
2521       u32 n_left_to_next;
2522
2523       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2524
2525       while (n_left_from > 0 && n_left_to_next > 0)
2526        {
2527           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2528           vlib_buffer_t *b0;
2529           u32 next0;
2530           u8 cached0 = 0;
2531           ip4_header_t *ip0;
2532           nat_reass_ip4_t *reass0;
2533           udp_header_t * udp0;
2534           tcp_header_t * tcp0;
2535           snat_session_key_t key0;
2536           clib_bihash_kv_8_8_t kv0, value0;
2537           snat_session_t * s0 = 0;
2538           u16 old_port0, new_port0;
2539           ip_csum_t sum0;
2540
2541           /* speculatively enqueue b0 to the current next frame */
2542           bi0 = from[0];
2543           to_next[0] = bi0;
2544           from += 1;
2545           to_next += 1;
2546           n_left_from -= 1;
2547           n_left_to_next -= 1;
2548
2549           b0 = vlib_get_buffer (vm, bi0);
2550           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2551
2552           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2553           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2554                                                                sw_if_index0);
2555
2556           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2557             {
2558               next0 = SNAT_IN2OUT_NEXT_DROP;
2559               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2560               goto trace0;
2561             }
2562
2563           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2564           udp0 = ip4_next_header (ip0);
2565           tcp0 = (tcp_header_t *) udp0;
2566           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2567
2568           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2569                                                  ip0->dst_address,
2570                                                  ip0->fragment_id,
2571                                                  ip0->protocol,
2572                                                  1,
2573                                                  &fragments_to_drop);
2574
2575           if (PREDICT_FALSE (!reass0))
2576             {
2577               next0 = SNAT_IN2OUT_NEXT_DROP;
2578               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2579               goto trace0;
2580             }
2581
2582           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2583             {
2584               key0.addr = ip0->src_address;
2585               key0.port = udp0->src_port;
2586               key0.protocol = proto0;
2587               key0.fib_index = rx_fib_index0;
2588               kv0.key = key0.as_u64;
2589
2590               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2591                 {
2592                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2593                       ip0, proto0, rx_fib_index0, thread_index)))
2594                     goto trace0;
2595
2596                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2597                                      &s0, node, next0, thread_index);
2598
2599                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2600                     goto trace0;
2601
2602                   reass0->sess_index = s0 - per_thread_data->sessions;
2603                 }
2604               else
2605                 {
2606                   s0 = pool_elt_at_index (per_thread_data->sessions,
2607                                           value0.value);
2608                   reass0->sess_index = value0.value;
2609                 }
2610               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2611             }
2612           else
2613             {
2614               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2615                 {
2616                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2617                     {
2618                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2619                       next0 = SNAT_IN2OUT_NEXT_DROP;
2620                       goto trace0;
2621                     }
2622                   cached0 = 1;
2623                   goto trace0;
2624                 }
2625               s0 = pool_elt_at_index (per_thread_data->sessions,
2626                                       reass0->sess_index);
2627             }
2628
2629           old_addr0 = ip0->src_address.as_u32;
2630           ip0->src_address = s0->out2in.addr;
2631           new_addr0 = ip0->src_address.as_u32;
2632           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2633
2634           sum0 = ip0->checksum;
2635           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2636                                  ip4_header_t,
2637                                  src_address /* changed member */);
2638           ip0->checksum = ip_csum_fold (sum0);
2639
2640           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2641             {
2642               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2643                 {
2644                   old_port0 = tcp0->src_port;
2645                   tcp0->src_port = s0->out2in.port;
2646                   new_port0 = tcp0->src_port;
2647
2648                   sum0 = tcp0->checksum;
2649                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2650                                          ip4_header_t,
2651                                          dst_address /* changed member */);
2652                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2653                                          ip4_header_t /* cheat */,
2654                                          length /* changed member */);
2655                   tcp0->checksum = ip_csum_fold(sum0);
2656                 }
2657               else
2658                 {
2659                   old_port0 = udp0->src_port;
2660                   udp0->src_port = s0->out2in.port;
2661                   udp0->checksum = 0;
2662                 }
2663             }
2664
2665           /* Hairpinning */
2666           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2667                                    s0->ext_host_port, proto0);
2668
2669           /* Accounting */
2670           s0->last_heard = now;
2671           s0->total_pkts++;
2672           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2673           /* Per-user LRU list maintenance */
2674           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2675                              s0->per_user_index);
2676           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2677                               s0->per_user_list_head_index,
2678                               s0->per_user_index);
2679
2680         trace0:
2681           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2682                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2683             {
2684               nat44_in2out_reass_trace_t *t =
2685                  vlib_add_trace (vm, node, b0, sizeof (*t));
2686               t->cached = cached0;
2687               t->sw_if_index = sw_if_index0;
2688               t->next_index = next0;
2689             }
2690
2691           if (cached0)
2692             {
2693               n_left_to_next++;
2694               to_next--;
2695             }
2696           else
2697             {
2698               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2699
2700               /* verify speculative enqueue, maybe switch current next frame */
2701               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2702                                                to_next, n_left_to_next,
2703                                                bi0, next0);
2704             }
2705
2706           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2707             {
2708               from = vlib_frame_vector_args (frame);
2709               u32 len = vec_len (fragments_to_loopback);
2710               if (len <= VLIB_FRAME_SIZE)
2711                 {
2712                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2713                   n_left_from = len;
2714                   vec_reset_length (fragments_to_loopback);
2715                 }
2716               else
2717                 {
2718                   clib_memcpy (from,
2719                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2720                                sizeof (u32) * VLIB_FRAME_SIZE);
2721                   n_left_from = VLIB_FRAME_SIZE;
2722                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2723                 }
2724             }
2725        }
2726
2727       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2728     }
2729
2730   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2731                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2732                                pkts_processed);
2733
2734   nat_send_all_to_node (vm, fragments_to_drop, node,
2735                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2736                         SNAT_IN2OUT_NEXT_DROP);
2737
2738   vec_free (fragments_to_drop);
2739   vec_free (fragments_to_loopback);
2740   return frame->n_vectors;
2741 }
2742
2743 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2744   .function = nat44_in2out_reass_node_fn,
2745   .name = "nat44-in2out-reass",
2746   .vector_size = sizeof (u32),
2747   .format_trace = format_nat44_in2out_reass_trace,
2748   .type = VLIB_NODE_TYPE_INTERNAL,
2749
2750   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2751   .error_strings = snat_in2out_error_strings,
2752
2753   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2754   .next_nodes = {
2755     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2756     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2757     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2758     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2759     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2760   },
2761 };
2762
2763 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2764                               nat44_in2out_reass_node_fn);
2765
2766 /**************************/
2767 /*** deterministic mode ***/
2768 /**************************/
2769 static uword
2770 snat_det_in2out_node_fn (vlib_main_t * vm,
2771                          vlib_node_runtime_t * node,
2772                          vlib_frame_t * frame)
2773 {
2774   u32 n_left_from, * from, * to_next;
2775   snat_in2out_next_t next_index;
2776   u32 pkts_processed = 0;
2777   snat_main_t * sm = &snat_main;
2778   u32 now = (u32) vlib_time_now (vm);
2779   u32 thread_index = vlib_get_thread_index ();
2780
2781   from = vlib_frame_vector_args (frame);
2782   n_left_from = frame->n_vectors;
2783   next_index = node->cached_next_index;
2784
2785   while (n_left_from > 0)
2786     {
2787       u32 n_left_to_next;
2788
2789       vlib_get_next_frame (vm, node, next_index,
2790                            to_next, n_left_to_next);
2791
2792       while (n_left_from >= 4 && n_left_to_next >= 2)
2793         {
2794           u32 bi0, bi1;
2795           vlib_buffer_t * b0, * b1;
2796           u32 next0, next1;
2797           u32 sw_if_index0, sw_if_index1;
2798           ip4_header_t * ip0, * ip1;
2799           ip_csum_t sum0, sum1;
2800           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2801           u16 old_port0, new_port0, lo_port0, i0;
2802           u16 old_port1, new_port1, lo_port1, i1;
2803           udp_header_t * udp0, * udp1;
2804           tcp_header_t * tcp0, * tcp1;
2805           u32 proto0, proto1;
2806           snat_det_out_key_t key0, key1;
2807           snat_det_map_t * dm0, * dm1;
2808           snat_det_session_t * ses0 = 0, * ses1 = 0;
2809           u32 rx_fib_index0, rx_fib_index1;
2810           icmp46_header_t * icmp0, * icmp1;
2811
2812           /* Prefetch next iteration. */
2813           {
2814             vlib_buffer_t * p2, * p3;
2815
2816             p2 = vlib_get_buffer (vm, from[2]);
2817             p3 = vlib_get_buffer (vm, from[3]);
2818
2819             vlib_prefetch_buffer_header (p2, LOAD);
2820             vlib_prefetch_buffer_header (p3, LOAD);
2821
2822             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2823             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2824           }
2825
2826           /* speculatively enqueue b0 and b1 to the current next frame */
2827           to_next[0] = bi0 = from[0];
2828           to_next[1] = bi1 = from[1];
2829           from += 2;
2830           to_next += 2;
2831           n_left_from -= 2;
2832           n_left_to_next -= 2;
2833
2834           b0 = vlib_get_buffer (vm, bi0);
2835           b1 = vlib_get_buffer (vm, bi1);
2836
2837           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2838           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2839
2840           ip0 = vlib_buffer_get_current (b0);
2841           udp0 = ip4_next_header (ip0);
2842           tcp0 = (tcp_header_t *) udp0;
2843
2844           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2845
2846           if (PREDICT_FALSE(ip0->ttl == 1))
2847             {
2848               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2849               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2850                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2851                                            0);
2852               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2853               goto trace0;
2854             }
2855
2856           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2857
2858           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2859             {
2860               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2861               icmp0 = (icmp46_header_t *) udp0;
2862
2863               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2864                                   rx_fib_index0, node, next0, thread_index,
2865                                   &ses0, &dm0);
2866               goto trace0;
2867             }
2868
2869           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2870           if (PREDICT_FALSE(!dm0))
2871             {
2872               clib_warning("no match for internal host %U",
2873                            format_ip4_address, &ip0->src_address);
2874               next0 = SNAT_IN2OUT_NEXT_DROP;
2875               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2876               goto trace0;
2877             }
2878
2879           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2880
2881           key0.ext_host_addr = ip0->dst_address;
2882           key0.ext_host_port = tcp0->dst;
2883
2884           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2885           if (PREDICT_FALSE(!ses0))
2886             {
2887               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2888                 {
2889                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2890                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2891
2892                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2893                     continue;
2894
2895                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2896                   break;
2897                 }
2898               if (PREDICT_FALSE(!ses0))
2899                 {
2900                   /* too many sessions for user, send ICMP error packet */
2901
2902                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2903                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2904                                                ICMP4_destination_unreachable_destination_unreachable_host,
2905                                                0);
2906                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2907                   goto trace0;
2908                 }
2909             }
2910
2911           new_port0 = ses0->out.out_port;
2912
2913           old_addr0.as_u32 = ip0->src_address.as_u32;
2914           ip0->src_address.as_u32 = new_addr0.as_u32;
2915           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2916
2917           sum0 = ip0->checksum;
2918           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2919                                  ip4_header_t,
2920                                  src_address /* changed member */);
2921           ip0->checksum = ip_csum_fold (sum0);
2922
2923           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2924             {
2925               if (tcp0->flags & TCP_FLAG_SYN)
2926                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2927               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2928                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2929               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2930                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2931               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2932                 snat_det_ses_close(dm0, ses0);
2933               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2934                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2935               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2936                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2937
2938               old_port0 = tcp0->src;
2939               tcp0->src = new_port0;
2940
2941               sum0 = tcp0->checksum;
2942               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2943                                      ip4_header_t,
2944                                      dst_address /* changed member */);
2945               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2946                                      ip4_header_t /* cheat */,
2947                                      length /* changed member */);
2948               tcp0->checksum = ip_csum_fold(sum0);
2949             }
2950           else
2951             {
2952               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2953               old_port0 = udp0->src_port;
2954               udp0->src_port = new_port0;
2955               udp0->checksum = 0;
2956             }
2957
2958           switch(ses0->state)
2959             {
2960             case SNAT_SESSION_UDP_ACTIVE:
2961                 ses0->expire = now + sm->udp_timeout;
2962                 break;
2963             case SNAT_SESSION_TCP_SYN_SENT:
2964             case SNAT_SESSION_TCP_FIN_WAIT:
2965             case SNAT_SESSION_TCP_CLOSE_WAIT:
2966             case SNAT_SESSION_TCP_LAST_ACK:
2967                 ses0->expire = now + sm->tcp_transitory_timeout;
2968                 break;
2969             case SNAT_SESSION_TCP_ESTABLISHED:
2970                 ses0->expire = now + sm->tcp_established_timeout;
2971                 break;
2972             }
2973
2974         trace0:
2975           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2976                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2977             {
2978               snat_in2out_trace_t *t =
2979                  vlib_add_trace (vm, node, b0, sizeof (*t));
2980               t->is_slow_path = 0;
2981               t->sw_if_index = sw_if_index0;
2982               t->next_index = next0;
2983               t->session_index = ~0;
2984               if (ses0)
2985                 t->session_index = ses0 - dm0->sessions;
2986             }
2987
2988           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2989
2990           ip1 = vlib_buffer_get_current (b1);
2991           udp1 = ip4_next_header (ip1);
2992           tcp1 = (tcp_header_t *) udp1;
2993
2994           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2995
2996           if (PREDICT_FALSE(ip1->ttl == 1))
2997             {
2998               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2999               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3000                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3001                                            0);
3002               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3003               goto trace1;
3004             }
3005
3006           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3007
3008           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3009             {
3010               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3011               icmp1 = (icmp46_header_t *) udp1;
3012
3013               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
3014                                   rx_fib_index1, node, next1, thread_index,
3015                                   &ses1, &dm1);
3016               goto trace1;
3017             }
3018
3019           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
3020           if (PREDICT_FALSE(!dm1))
3021             {
3022               clib_warning("no match for internal host %U",
3023                            format_ip4_address, &ip0->src_address);
3024               next1 = SNAT_IN2OUT_NEXT_DROP;
3025               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3026               goto trace1;
3027             }
3028
3029           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3030
3031           key1.ext_host_addr = ip1->dst_address;
3032           key1.ext_host_port = tcp1->dst;
3033
3034           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3035           if (PREDICT_FALSE(!ses1))
3036             {
3037               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3038                 {
3039                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
3040                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3041
3042                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3043                     continue;
3044
3045                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3046                   break;
3047                 }
3048               if (PREDICT_FALSE(!ses1))
3049                 {
3050                   /* too many sessions for user, send ICMP error packet */
3051
3052                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3053                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3054                                                ICMP4_destination_unreachable_destination_unreachable_host,
3055                                                0);
3056                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3057                   goto trace1;
3058                 }
3059             }
3060
3061           new_port1 = ses1->out.out_port;
3062
3063           old_addr1.as_u32 = ip1->src_address.as_u32;
3064           ip1->src_address.as_u32 = new_addr1.as_u32;
3065           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3066
3067           sum1 = ip1->checksum;
3068           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3069                                  ip4_header_t,
3070                                  src_address /* changed member */);
3071           ip1->checksum = ip_csum_fold (sum1);
3072
3073           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3074             {
3075               if (tcp1->flags & TCP_FLAG_SYN)
3076                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3077               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3078                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3079               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3080                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3081               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3082                 snat_det_ses_close(dm1, ses1);
3083               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3084                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3085               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3086                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3087
3088               old_port1 = tcp1->src;
3089               tcp1->src = new_port1;
3090
3091               sum1 = tcp1->checksum;
3092               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3093                                      ip4_header_t,
3094                                      dst_address /* changed member */);
3095               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3096                                      ip4_header_t /* cheat */,
3097                                      length /* changed member */);
3098               tcp1->checksum = ip_csum_fold(sum1);
3099             }
3100           else
3101             {
3102               ses1->state = SNAT_SESSION_UDP_ACTIVE;
3103               old_port1 = udp1->src_port;
3104               udp1->src_port = new_port1;
3105               udp1->checksum = 0;
3106             }
3107
3108           switch(ses1->state)
3109             {
3110             case SNAT_SESSION_UDP_ACTIVE:
3111                 ses1->expire = now + sm->udp_timeout;
3112                 break;
3113             case SNAT_SESSION_TCP_SYN_SENT:
3114             case SNAT_SESSION_TCP_FIN_WAIT:
3115             case SNAT_SESSION_TCP_CLOSE_WAIT:
3116             case SNAT_SESSION_TCP_LAST_ACK:
3117                 ses1->expire = now + sm->tcp_transitory_timeout;
3118                 break;
3119             case SNAT_SESSION_TCP_ESTABLISHED:
3120                 ses1->expire = now + sm->tcp_established_timeout;
3121                 break;
3122             }
3123
3124         trace1:
3125           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3126                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3127             {
3128               snat_in2out_trace_t *t =
3129                  vlib_add_trace (vm, node, b1, sizeof (*t));
3130               t->is_slow_path = 0;
3131               t->sw_if_index = sw_if_index1;
3132               t->next_index = next1;
3133               t->session_index = ~0;
3134               if (ses1)
3135                 t->session_index = ses1 - dm1->sessions;
3136             }
3137
3138           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3139
3140           /* verify speculative enqueues, maybe switch current next frame */
3141           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3142                                            to_next, n_left_to_next,
3143                                            bi0, bi1, next0, next1);
3144          }
3145
3146       while (n_left_from > 0 && n_left_to_next > 0)
3147         {
3148           u32 bi0;
3149           vlib_buffer_t * b0;
3150           u32 next0;
3151           u32 sw_if_index0;
3152           ip4_header_t * ip0;
3153           ip_csum_t sum0;
3154           ip4_address_t new_addr0, old_addr0;
3155           u16 old_port0, new_port0, lo_port0, i0;
3156           udp_header_t * udp0;
3157           tcp_header_t * tcp0;
3158           u32 proto0;
3159           snat_det_out_key_t key0;
3160           snat_det_map_t * dm0;
3161           snat_det_session_t * ses0 = 0;
3162           u32 rx_fib_index0;
3163           icmp46_header_t * icmp0;
3164
3165           /* speculatively enqueue b0 to the current next frame */
3166           bi0 = from[0];
3167           to_next[0] = bi0;
3168           from += 1;
3169           to_next += 1;
3170           n_left_from -= 1;
3171           n_left_to_next -= 1;
3172
3173           b0 = vlib_get_buffer (vm, bi0);
3174           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3175
3176           ip0 = vlib_buffer_get_current (b0);
3177           udp0 = ip4_next_header (ip0);
3178           tcp0 = (tcp_header_t *) udp0;
3179
3180           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3181
3182           if (PREDICT_FALSE(ip0->ttl == 1))
3183             {
3184               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3185               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3186                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3187                                            0);
3188               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3189               goto trace00;
3190             }
3191
3192           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3193
3194           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3195             {
3196               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3197               icmp0 = (icmp46_header_t *) udp0;
3198
3199               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3200                                   rx_fib_index0, node, next0, thread_index,
3201                                   &ses0, &dm0);
3202               goto trace00;
3203             }
3204
3205           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3206           if (PREDICT_FALSE(!dm0))
3207             {
3208               clib_warning("no match for internal host %U",
3209                            format_ip4_address, &ip0->src_address);
3210               next0 = SNAT_IN2OUT_NEXT_DROP;
3211               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3212               goto trace00;
3213             }
3214
3215           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3216
3217           key0.ext_host_addr = ip0->dst_address;
3218           key0.ext_host_port = tcp0->dst;
3219
3220           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3221           if (PREDICT_FALSE(!ses0))
3222             {
3223               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3224                 {
3225                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3226                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3227
3228                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3229                     continue;
3230
3231                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3232                   break;
3233                 }
3234               if (PREDICT_FALSE(!ses0))
3235                 {
3236                   /* too many sessions for user, send ICMP error packet */
3237
3238                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3239                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3240                                                ICMP4_destination_unreachable_destination_unreachable_host,
3241                                                0);
3242                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3243                   goto trace00;
3244                 }
3245             }
3246
3247           new_port0 = ses0->out.out_port;
3248
3249           old_addr0.as_u32 = ip0->src_address.as_u32;
3250           ip0->src_address.as_u32 = new_addr0.as_u32;
3251           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3252
3253           sum0 = ip0->checksum;
3254           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3255                                  ip4_header_t,
3256                                  src_address /* changed member */);
3257           ip0->checksum = ip_csum_fold (sum0);
3258
3259           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3260             {
3261               if (tcp0->flags & TCP_FLAG_SYN)
3262                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3263               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3264                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3265               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3266                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3267               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3268                 snat_det_ses_close(dm0, ses0);
3269               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3270                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3271               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3272                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3273
3274               old_port0 = tcp0->src;
3275               tcp0->src = new_port0;
3276
3277               sum0 = tcp0->checksum;
3278               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3279                                      ip4_header_t,
3280                                      dst_address /* changed member */);
3281               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3282                                      ip4_header_t /* cheat */,
3283                                      length /* changed member */);
3284               tcp0->checksum = ip_csum_fold(sum0);
3285             }
3286           else
3287             {
3288               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3289               old_port0 = udp0->src_port;
3290               udp0->src_port = new_port0;
3291               udp0->checksum = 0;
3292             }
3293
3294           switch(ses0->state)
3295             {
3296             case SNAT_SESSION_UDP_ACTIVE:
3297                 ses0->expire = now + sm->udp_timeout;
3298                 break;
3299             case SNAT_SESSION_TCP_SYN_SENT:
3300             case SNAT_SESSION_TCP_FIN_WAIT:
3301             case SNAT_SESSION_TCP_CLOSE_WAIT:
3302             case SNAT_SESSION_TCP_LAST_ACK:
3303                 ses0->expire = now + sm->tcp_transitory_timeout;
3304                 break;
3305             case SNAT_SESSION_TCP_ESTABLISHED:
3306                 ses0->expire = now + sm->tcp_established_timeout;
3307                 break;
3308             }
3309
3310         trace00:
3311           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3312                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3313             {
3314               snat_in2out_trace_t *t =
3315                  vlib_add_trace (vm, node, b0, sizeof (*t));
3316               t->is_slow_path = 0;
3317               t->sw_if_index = sw_if_index0;
3318               t->next_index = next0;
3319               t->session_index = ~0;
3320               if (ses0)
3321                 t->session_index = ses0 - dm0->sessions;
3322             }
3323
3324           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3325
3326           /* verify speculative enqueue, maybe switch current next frame */
3327           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3328                                            to_next, n_left_to_next,
3329                                            bi0, next0);
3330         }
3331
3332       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3333     }
3334
3335   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3336                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3337                                pkts_processed);
3338   return frame->n_vectors;
3339 }
3340
3341 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3342   .function = snat_det_in2out_node_fn,
3343   .name = "nat44-det-in2out",
3344   .vector_size = sizeof (u32),
3345   .format_trace = format_snat_in2out_trace,
3346   .type = VLIB_NODE_TYPE_INTERNAL,
3347
3348   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3349   .error_strings = snat_in2out_error_strings,
3350
3351   .runtime_data_bytes = sizeof (snat_runtime_t),
3352
3353   .n_next_nodes = 3,
3354
3355   /* edit / add dispositions here */
3356   .next_nodes = {
3357     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3358     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3359     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3360   },
3361 };
3362
3363 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3364
3365 /**
3366  * Get address and port values to be used for ICMP packet translation
3367  * and create session if needed
3368  *
3369  * @param[in,out] sm             NAT main
3370  * @param[in,out] node           NAT node runtime
3371  * @param[in] thread_index       thread index
3372  * @param[in,out] b0             buffer containing packet to be translated
3373  * @param[out] p_proto           protocol used for matching
3374  * @param[out] p_value           address and port after NAT translation
3375  * @param[out] p_dont_translate  if packet should not be translated
3376  * @param d                      optional parameter
3377  * @param e                      optional parameter
3378  */
3379 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3380                           u32 thread_index, vlib_buffer_t *b0,
3381                           ip4_header_t *ip0, u8 *p_proto,
3382                           snat_session_key_t *p_value,
3383                           u8 *p_dont_translate, void *d, void *e)
3384 {
3385   icmp46_header_t *icmp0;
3386   u32 sw_if_index0;
3387   u32 rx_fib_index0;
3388   u8 protocol;
3389   snat_det_out_key_t key0;
3390   u8 dont_translate = 0;
3391   u32 next0 = ~0;
3392   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3393   ip4_header_t *inner_ip0;
3394   void *l4_header = 0;
3395   icmp46_header_t *inner_icmp0;
3396   snat_det_map_t * dm0 = 0;
3397   ip4_address_t new_addr0;
3398   u16 lo_port0, i0;
3399   snat_det_session_t * ses0 = 0;
3400   ip4_address_t in_addr;
3401   u16 in_port;
3402
3403   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3404   echo0 = (icmp_echo_header_t *)(icmp0+1);
3405   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3406   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3407
3408   if (!icmp_is_error_message (icmp0))
3409     {
3410       protocol = SNAT_PROTOCOL_ICMP;
3411       in_addr = ip0->src_address;
3412       in_port = echo0->identifier;
3413     }
3414   else
3415     {
3416       inner_ip0 = (ip4_header_t *)(echo0+1);
3417       l4_header = ip4_next_header (inner_ip0);
3418       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3419       in_addr = inner_ip0->dst_address;
3420       switch (protocol)
3421         {
3422         case SNAT_PROTOCOL_ICMP:
3423           inner_icmp0 = (icmp46_header_t*)l4_header;
3424           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3425           in_port = inner_echo0->identifier;
3426           break;
3427         case SNAT_PROTOCOL_UDP:
3428         case SNAT_PROTOCOL_TCP:
3429           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3430           break;
3431         default:
3432           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3433           next0 = SNAT_IN2OUT_NEXT_DROP;
3434           goto out;
3435         }
3436     }
3437
3438   dm0 = snat_det_map_by_user(sm, &in_addr);
3439   if (PREDICT_FALSE(!dm0))
3440     {
3441       clib_warning("no match for internal host %U",
3442                    format_ip4_address, &in_addr);
3443       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3444           IP_PROTOCOL_ICMP, rx_fib_index0)))
3445         {
3446           dont_translate = 1;
3447           goto out;
3448         }
3449       next0 = SNAT_IN2OUT_NEXT_DROP;
3450       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3451       goto out;
3452     }
3453
3454   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3455
3456   key0.ext_host_addr = ip0->dst_address;
3457   key0.ext_host_port = 0;
3458
3459   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3460   if (PREDICT_FALSE(!ses0))
3461     {
3462       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3463           IP_PROTOCOL_ICMP, rx_fib_index0)))
3464         {
3465           dont_translate = 1;
3466           goto out;
3467         }
3468       if (icmp0->type != ICMP4_echo_request)
3469         {
3470           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3471           next0 = SNAT_IN2OUT_NEXT_DROP;
3472           goto out;
3473         }
3474       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3475         {
3476           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3477             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3478
3479           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3480             continue;
3481
3482           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3483           break;
3484         }
3485       if (PREDICT_FALSE(!ses0))
3486         {
3487           next0 = SNAT_IN2OUT_NEXT_DROP;
3488           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3489           goto out;
3490         }
3491     }
3492
3493   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3494                     !icmp_is_error_message (icmp0)))
3495     {
3496       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3497       next0 = SNAT_IN2OUT_NEXT_DROP;
3498       goto out;
3499     }
3500
3501   u32 now = (u32) vlib_time_now (sm->vlib_main);
3502
3503   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3504   ses0->expire = now + sm->icmp_timeout;
3505
3506 out:
3507   *p_proto = protocol;
3508   if (ses0)
3509     {
3510       p_value->addr = new_addr0;
3511       p_value->fib_index = sm->outside_fib_index;
3512       p_value->port = ses0->out.out_port;
3513     }
3514   *p_dont_translate = dont_translate;
3515   if (d)
3516     *(snat_det_session_t**)d = ses0;
3517   if (e)
3518     *(snat_det_map_t**)e = dm0;
3519   return next0;
3520 }
3521
3522 /**********************/
3523 /*** worker handoff ***/
3524 /**********************/
3525 static inline uword
3526 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3527                                       vlib_node_runtime_t * node,
3528                                       vlib_frame_t * frame,
3529                                       u8 is_output)
3530 {
3531   snat_main_t *sm = &snat_main;
3532   vlib_thread_main_t *tm = vlib_get_thread_main ();
3533   u32 n_left_from, *from, *to_next = 0;
3534   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3535   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3536     = 0;
3537   vlib_frame_queue_elt_t *hf = 0;
3538   vlib_frame_t *f = 0;
3539   int i;
3540   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3541   u32 next_worker_index = 0;
3542   u32 current_worker_index = ~0;
3543   u32 thread_index = vlib_get_thread_index ();
3544   u32 fq_index;
3545   u32 to_node_index;
3546
3547   ASSERT (vec_len (sm->workers));
3548
3549   if (is_output)
3550     {
3551       fq_index = sm->fq_in2out_output_index;
3552       to_node_index = sm->in2out_output_node_index;
3553     }
3554   else
3555     {
3556       fq_index = sm->fq_in2out_index;
3557       to_node_index = sm->in2out_node_index;
3558     }
3559
3560   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3561     {
3562       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3563
3564       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3565                                sm->first_worker_index + sm->num_workers - 1,
3566                                (vlib_frame_queue_t *) (~0));
3567     }
3568
3569   from = vlib_frame_vector_args (frame);
3570   n_left_from = frame->n_vectors;
3571
3572   while (n_left_from > 0)
3573     {
3574       u32 bi0;
3575       vlib_buffer_t *b0;
3576       u32 sw_if_index0;
3577       u32 rx_fib_index0;
3578       ip4_header_t * ip0;
3579       u8 do_handoff;
3580
3581       bi0 = from[0];
3582       from += 1;
3583       n_left_from -= 1;
3584
3585       b0 = vlib_get_buffer (vm, bi0);
3586
3587       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3588       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3589
3590       ip0 = vlib_buffer_get_current (b0);
3591
3592       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3593
3594       if (PREDICT_FALSE (next_worker_index != thread_index))
3595         {
3596           do_handoff = 1;
3597
3598           if (next_worker_index != current_worker_index)
3599             {
3600               if (hf)
3601                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3602
3603               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3604                                                       next_worker_index,
3605                                                       handoff_queue_elt_by_worker_index);
3606
3607               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3608               to_next_worker = &hf->buffer_index[hf->n_vectors];
3609               current_worker_index = next_worker_index;
3610             }
3611
3612           /* enqueue to correct worker thread */
3613           to_next_worker[0] = bi0;
3614           to_next_worker++;
3615           n_left_to_next_worker--;
3616
3617           if (n_left_to_next_worker == 0)
3618             {
3619               hf->n_vectors = VLIB_FRAME_SIZE;
3620               vlib_put_frame_queue_elt (hf);
3621               current_worker_index = ~0;
3622               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3623               hf = 0;
3624             }
3625         }
3626       else
3627         {
3628           do_handoff = 0;
3629           /* if this is 1st frame */
3630           if (!f)
3631             {
3632               f = vlib_get_frame_to_node (vm, to_node_index);
3633               to_next = vlib_frame_vector_args (f);
3634             }
3635
3636           to_next[0] = bi0;
3637           to_next += 1;
3638           f->n_vectors++;
3639         }
3640
3641       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3642                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3643         {
3644           snat_in2out_worker_handoff_trace_t *t =
3645             vlib_add_trace (vm, node, b0, sizeof (*t));
3646           t->next_worker_index = next_worker_index;
3647           t->do_handoff = do_handoff;
3648         }
3649     }
3650
3651   if (f)
3652     vlib_put_frame_to_node (vm, to_node_index, f);
3653
3654   if (hf)
3655     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3656
3657   /* Ship frames to the worker nodes */
3658   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3659     {
3660       if (handoff_queue_elt_by_worker_index[i])
3661         {
3662           hf = handoff_queue_elt_by_worker_index[i];
3663           /*
3664            * It works better to let the handoff node
3665            * rate-adapt, always ship the handoff queue element.
3666            */
3667           if (1 || hf->n_vectors == hf->last_n_vectors)
3668             {
3669               vlib_put_frame_queue_elt (hf);
3670               handoff_queue_elt_by_worker_index[i] = 0;
3671             }
3672           else
3673             hf->last_n_vectors = hf->n_vectors;
3674         }
3675       congested_handoff_queue_by_worker_index[i] =
3676         (vlib_frame_queue_t *) (~0);
3677     }
3678   hf = 0;
3679   current_worker_index = ~0;
3680   return frame->n_vectors;
3681 }
3682
3683 static uword
3684 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3685                                vlib_node_runtime_t * node,
3686                                vlib_frame_t * frame)
3687 {
3688   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3689 }
3690
3691 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3692   .function = snat_in2out_worker_handoff_fn,
3693   .name = "nat44-in2out-worker-handoff",
3694   .vector_size = sizeof (u32),
3695   .format_trace = format_snat_in2out_worker_handoff_trace,
3696   .type = VLIB_NODE_TYPE_INTERNAL,
3697
3698   .n_next_nodes = 1,
3699
3700   .next_nodes = {
3701     [0] = "error-drop",
3702   },
3703 };
3704
3705 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3706                               snat_in2out_worker_handoff_fn);
3707
3708 static uword
3709 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3710                                       vlib_node_runtime_t * node,
3711                                       vlib_frame_t * frame)
3712 {
3713   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3714 }
3715
3716 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3717   .function = snat_in2out_output_worker_handoff_fn,
3718   .name = "nat44-in2out-output-worker-handoff",
3719   .vector_size = sizeof (u32),
3720   .format_trace = format_snat_in2out_worker_handoff_trace,
3721   .type = VLIB_NODE_TYPE_INTERNAL,
3722
3723   .n_next_nodes = 1,
3724
3725   .next_nodes = {
3726     [0] = "error-drop",
3727   },
3728 };
3729
3730 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3731                               snat_in2out_output_worker_handoff_fn);
3732
3733 static_always_inline int
3734 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3735 {
3736   snat_address_t * ap;
3737   clib_bihash_kv_8_8_t kv, value;
3738   snat_session_key_t m_key;
3739
3740   vec_foreach (ap, sm->addresses)
3741     {
3742       if (ap->addr.as_u32 == dst_addr->as_u32)
3743         return 1;
3744     }
3745
3746   m_key.addr.as_u32 = dst_addr->as_u32;
3747   m_key.fib_index = sm->outside_fib_index;
3748   m_key.port = 0;
3749   m_key.protocol = 0;
3750   kv.key = m_key.as_u64;
3751   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3752     return 1;
3753
3754   return 0;
3755 }
3756
3757 static uword
3758 snat_hairpin_dst_fn (vlib_main_t * vm,
3759                      vlib_node_runtime_t * node,
3760                      vlib_frame_t * frame)
3761 {
3762   u32 n_left_from, * from, * to_next;
3763   snat_in2out_next_t next_index;
3764   u32 pkts_processed = 0;
3765   snat_main_t * sm = &snat_main;
3766
3767   from = vlib_frame_vector_args (frame);
3768   n_left_from = frame->n_vectors;
3769   next_index = node->cached_next_index;
3770
3771   while (n_left_from > 0)
3772     {
3773       u32 n_left_to_next;
3774
3775       vlib_get_next_frame (vm, node, next_index,
3776                            to_next, n_left_to_next);
3777
3778       while (n_left_from > 0 && n_left_to_next > 0)
3779         {
3780           u32 bi0;
3781           vlib_buffer_t * b0;
3782           u32 next0;
3783           ip4_header_t * ip0;
3784           u32 proto0;
3785
3786           /* speculatively enqueue b0 to the current next frame */
3787           bi0 = from[0];
3788           to_next[0] = bi0;
3789           from += 1;
3790           to_next += 1;
3791           n_left_from -= 1;
3792           n_left_to_next -= 1;
3793
3794           b0 = vlib_get_buffer (vm, bi0);
3795           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3796           ip0 = vlib_buffer_get_current (b0);
3797
3798           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3799
3800           vnet_buffer (b0)->snat.flags = 0;
3801           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3802             {
3803               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3804                 {
3805                   udp_header_t * udp0 = ip4_next_header (ip0);
3806                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3807
3808                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3809                 }
3810               else if (proto0 == SNAT_PROTOCOL_ICMP)
3811                 {
3812                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3813
3814                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3815                 }
3816               else
3817                 {
3818                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3819                 }
3820
3821               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3822             }
3823
3824           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3825
3826           /* verify speculative enqueue, maybe switch current next frame */
3827           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3828                                            to_next, n_left_to_next,
3829                                            bi0, next0);
3830          }
3831
3832       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3833     }
3834
3835   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3836                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3837                                pkts_processed);
3838   return frame->n_vectors;
3839 }
3840
3841 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3842   .function = snat_hairpin_dst_fn,
3843   .name = "nat44-hairpin-dst",
3844   .vector_size = sizeof (u32),
3845   .type = VLIB_NODE_TYPE_INTERNAL,
3846   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3847   .error_strings = snat_in2out_error_strings,
3848   .n_next_nodes = 2,
3849   .next_nodes = {
3850     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3851     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3852   },
3853 };
3854
3855 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3856                               snat_hairpin_dst_fn);
3857
3858 static uword
3859 snat_hairpin_src_fn (vlib_main_t * vm,
3860                      vlib_node_runtime_t * node,
3861                      vlib_frame_t * frame)
3862 {
3863   u32 n_left_from, * from, * to_next;
3864   snat_in2out_next_t next_index;
3865   u32 pkts_processed = 0;
3866   snat_main_t *sm = &snat_main;
3867
3868   from = vlib_frame_vector_args (frame);
3869   n_left_from = frame->n_vectors;
3870   next_index = node->cached_next_index;
3871
3872   while (n_left_from > 0)
3873     {
3874       u32 n_left_to_next;
3875
3876       vlib_get_next_frame (vm, node, next_index,
3877                            to_next, n_left_to_next);
3878
3879       while (n_left_from > 0 && n_left_to_next > 0)
3880         {
3881           u32 bi0;
3882           vlib_buffer_t * b0;
3883           u32 next0;
3884           snat_interface_t *i;
3885           u32 sw_if_index0;
3886
3887           /* speculatively enqueue b0 to the current next frame */
3888           bi0 = from[0];
3889           to_next[0] = bi0;
3890           from += 1;
3891           to_next += 1;
3892           n_left_from -= 1;
3893           n_left_to_next -= 1;
3894
3895           b0 = vlib_get_buffer (vm, bi0);
3896           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3897           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3898
3899           pool_foreach (i, sm->output_feature_interfaces,
3900           ({
3901             /* Only packets from NAT inside interface */
3902             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3903               {
3904                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3905                                     SNAT_FLAG_HAIRPINNING))
3906                   {
3907                     if (PREDICT_TRUE (sm->num_workers > 1))
3908                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3909                     else
3910                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3911                   }
3912                 break;
3913               }
3914           }));
3915
3916           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3917
3918           /* verify speculative enqueue, maybe switch current next frame */
3919           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3920                                            to_next, n_left_to_next,
3921                                            bi0, next0);
3922          }
3923
3924       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3925     }
3926
3927   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3928                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3929                                pkts_processed);
3930   return frame->n_vectors;
3931 }
3932
3933 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3934   .function = snat_hairpin_src_fn,
3935   .name = "nat44-hairpin-src",
3936   .vector_size = sizeof (u32),
3937   .type = VLIB_NODE_TYPE_INTERNAL,
3938   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3939   .error_strings = snat_in2out_error_strings,
3940   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3941   .next_nodes = {
3942      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3943      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3944      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3945      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3946   },
3947 };
3948
3949 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3950                               snat_hairpin_src_fn);
3951
3952 static uword
3953 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3954                                 vlib_node_runtime_t * node,
3955                                 vlib_frame_t * frame)
3956 {
3957   u32 n_left_from, * from, * to_next;
3958   snat_in2out_next_t next_index;
3959   u32 pkts_processed = 0;
3960   snat_main_t * sm = &snat_main;
3961   u32 stats_node_index;
3962
3963   stats_node_index = snat_in2out_fast_node.index;
3964
3965   from = vlib_frame_vector_args (frame);
3966   n_left_from = frame->n_vectors;
3967   next_index = node->cached_next_index;
3968
3969   while (n_left_from > 0)
3970     {
3971       u32 n_left_to_next;
3972
3973       vlib_get_next_frame (vm, node, next_index,
3974                            to_next, n_left_to_next);
3975
3976       while (n_left_from > 0 && n_left_to_next > 0)
3977         {
3978           u32 bi0;
3979           vlib_buffer_t * b0;
3980           u32 next0;
3981           u32 sw_if_index0;
3982           ip4_header_t * ip0;
3983           ip_csum_t sum0;
3984           u32 new_addr0, old_addr0;
3985           u16 old_port0, new_port0;
3986           udp_header_t * udp0;
3987           tcp_header_t * tcp0;
3988           icmp46_header_t * icmp0;
3989           snat_session_key_t key0, sm0;
3990           u32 proto0;
3991           u32 rx_fib_index0;
3992
3993           /* speculatively enqueue b0 to the current next frame */
3994           bi0 = from[0];
3995           to_next[0] = bi0;
3996           from += 1;
3997           to_next += 1;
3998           n_left_from -= 1;
3999           n_left_to_next -= 1;
4000
4001           b0 = vlib_get_buffer (vm, bi0);
4002           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4003
4004           ip0 = vlib_buffer_get_current (b0);
4005           udp0 = ip4_next_header (ip0);
4006           tcp0 = (tcp_header_t *) udp0;
4007           icmp0 = (icmp46_header_t *) udp0;
4008
4009           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4010           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4011
4012           if (PREDICT_FALSE(ip0->ttl == 1))
4013             {
4014               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4015               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4016                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4017                                            0);
4018               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4019               goto trace0;
4020             }
4021
4022           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4023
4024           if (PREDICT_FALSE (proto0 == ~0))
4025               goto trace0;
4026
4027           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4028             {
4029               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4030                                   rx_fib_index0, node, next0, ~0, 0, 0);
4031               goto trace0;
4032             }
4033
4034           key0.addr = ip0->src_address;
4035           key0.protocol = proto0;
4036           key0.port = udp0->src_port;
4037           key0.fib_index = rx_fib_index0;
4038
4039           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0))
4040             {
4041               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4042               next0= SNAT_IN2OUT_NEXT_DROP;
4043               goto trace0;
4044             }
4045
4046           new_addr0 = sm0.addr.as_u32;
4047           new_port0 = sm0.port;
4048           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4049           old_addr0 = ip0->src_address.as_u32;
4050           ip0->src_address.as_u32 = new_addr0;
4051
4052           sum0 = ip0->checksum;
4053           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4054                                  ip4_header_t,
4055                                  src_address /* changed member */);
4056           ip0->checksum = ip_csum_fold (sum0);
4057
4058           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4059             {
4060               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4061                 {
4062                   old_port0 = tcp0->src_port;
4063                   tcp0->src_port = new_port0;
4064
4065                   sum0 = tcp0->checksum;
4066                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4067                                          ip4_header_t,
4068                                          dst_address /* changed member */);
4069                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4070                                          ip4_header_t /* cheat */,
4071                                          length /* changed member */);
4072                   tcp0->checksum = ip_csum_fold(sum0);
4073                 }
4074               else
4075                 {
4076                   old_port0 = udp0->src_port;
4077                   udp0->src_port = new_port0;
4078                   udp0->checksum = 0;
4079                 }
4080             }
4081           else
4082             {
4083               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4084                 {
4085                   sum0 = tcp0->checksum;
4086                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4087                                          ip4_header_t,
4088                                          dst_address /* changed member */);
4089                   tcp0->checksum = ip_csum_fold(sum0);
4090                 }
4091             }
4092
4093           /* Hairpinning */
4094           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4095
4096         trace0:
4097           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4098                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4099             {
4100               snat_in2out_trace_t *t =
4101                  vlib_add_trace (vm, node, b0, sizeof (*t));
4102               t->sw_if_index = sw_if_index0;
4103               t->next_index = next0;
4104             }
4105
4106           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4107
4108           /* verify speculative enqueue, maybe switch current next frame */
4109           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4110                                            to_next, n_left_to_next,
4111                                            bi0, next0);
4112         }
4113
4114       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4115     }
4116
4117   vlib_node_increment_counter (vm, stats_node_index,
4118                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4119                                pkts_processed);
4120   return frame->n_vectors;
4121 }
4122
4123
4124 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
4125   .function = snat_in2out_fast_static_map_fn,
4126   .name = "nat44-in2out-fast",
4127   .vector_size = sizeof (u32),
4128   .format_trace = format_snat_in2out_fast_trace,
4129   .type = VLIB_NODE_TYPE_INTERNAL,
4130
4131   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4132   .error_strings = snat_in2out_error_strings,
4133
4134   .runtime_data_bytes = sizeof (snat_runtime_t),
4135
4136   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4137
4138   /* edit / add dispositions here */
4139   .next_nodes = {
4140     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4141     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4142     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4143     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4144     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4145   },
4146 };
4147
4148 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);