src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37   u32 is_slow_path;
  38 } snat_in2out_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_in2out_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  51   char * tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  66
  67   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  68               t->sw_if_index, t->next_index);
  69
  70   return s;
  71 }
  72
  73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   snat_in2out_worker_handoff_trace_t * t =
  78     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  79   char * m;
  80
  81   m = t->do_handoff ? "next worker" : "same worker";
  82   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  83
  84   return s;
  85 }
  86
  87 typedef struct {
  88   u32 sw_if_index;
  89   u32 next_index;
  90   u8 cached;
  91 } nat44_in2out_reass_trace_t;
  92
  93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  94 {
  95   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  96   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  97   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  98
  99   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 100               t->sw_if_index, t->next_index,
 101               t->cached ? "cached" : "translated");
 102
 103   return s;
 104 }
 105
 106 vlib_node_registration_t snat_in2out_node;
 107 vlib_node_registration_t snat_in2out_slowpath_node;
 108 vlib_node_registration_t snat_in2out_fast_node;
 109 vlib_node_registration_t snat_in2out_worker_handoff_node;
 110 vlib_node_registration_t snat_det_in2out_node;
 111 vlib_node_registration_t snat_in2out_output_node;
 112 vlib_node_registration_t snat_in2out_output_slowpath_node;
 113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 114 vlib_node_registration_t snat_hairpin_dst_node;
 115 vlib_node_registration_t snat_hairpin_src_node;
 116 vlib_node_registration_t nat44_hairpinning_node;
 117 vlib_node_registration_t nat44_in2out_reass_node;
 118
 119
 120 #define foreach_snat_in2out_error                       \
 121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 122 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 123 _(OUT_OF_PORTS, "Out of ports")                         \
 124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 125 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 126 _(NO_TRANSLATION, "No translation")                     \
 127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 128 _(DROP_FRAGMENT, "Drop fragment")                       \
 129 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 131
 132 typedef enum {
 133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 134   foreach_snat_in2out_error
 135 #undef _
 136   SNAT_IN2OUT_N_ERROR,
 137 } snat_in2out_error_t;
 138
 139 static char * snat_in2out_error_strings[] = {
 140 #define _(sym,string) string,
 141   foreach_snat_in2out_error
 142 #undef _
 143 };
 144
 145 typedef enum {
 146   SNAT_IN2OUT_NEXT_LOOKUP,
 147   SNAT_IN2OUT_NEXT_DROP,
 148   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 149   SNAT_IN2OUT_NEXT_SLOW_PATH,
 150   SNAT_IN2OUT_NEXT_REASS,
 151   SNAT_IN2OUT_N_NEXT,
 152 } snat_in2out_next_t;
 153
 154 typedef enum {
 155   SNAT_HAIRPIN_SRC_NEXT_DROP,
 156   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 157   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 158   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 159   SNAT_HAIRPIN_SRC_N_NEXT,
 160 } snat_hairpin_next_t;
 161
 162 /**
 163  * @brief Check if packet should be translated
 164  *
 165  * Packets aimed at outside interface and external address with active session
 166  * should be translated.
 167  *
 168  * @param sm            NAT main
 169  * @param rt            NAT runtime data
 170  * @param sw_if_index0  index of the inside interface
 171  * @param ip0           IPv4 header
 172  * @param proto0        NAT protocol
 173  * @param rx_fib_index0 RX FIB index
 174  *
 175  * @returns 0 if packet should be translated otherwise 1
 176  */
 177 static inline int
 178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 179                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                          u32 rx_fib_index0)
 181 {
 182   if (sm->out2in_dpo)
 183     return 0;
 184
 185   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 186   fib_prefix_t pfx = {
 187     .fp_proto = FIB_PROTOCOL_IP4,
 188     .fp_len = 32,
 189     .fp_addr = {
 190         .ip4.as_u32 = ip0->dst_address.as_u32,
 191     },
 192   };
 193
 194   /* Don't NAT packet aimed at the intfc address */
 195   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 196                                       ip0->dst_address.as_u32)))
 197     return 1;
 198
 199   fei = fib_table_lookup (rx_fib_index0, &pfx);
 200   if (FIB_NODE_INDEX_INVALID != fei)
 201     {
 202       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 203       if (sw_if_index == ~0)
 204         {
 205           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 206           if (FIB_NODE_INDEX_INVALID != fei)
 207             sw_if_index = fib_entry_get_resolving_interface (fei);
 208         }
 209       snat_interface_t *i;
 210       pool_foreach (i, sm->interfaces,
 211       ({
 212         /* NAT packet aimed at outside interface */
 213         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 214           return 0;
 215       }));
 216     }
 217
 218   return 1;
 219 }
 220
 221 static inline int
 222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 223                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 224                     u32 rx_fib_index0, u32 thread_index)
 225 {
 226   udp_header_t * udp0 = ip4_next_header (ip0);
 227   snat_session_key_t key0, sm0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229
 230   key0.addr = ip0->dst_address;
 231   key0.port = udp0->dst_port;
 232   key0.protocol = proto0;
 233   key0.fib_index = sm->outside_fib_index;
 234   kv0.key = key0.as_u64;
 235
 236   /* NAT packet aimed at external address if */
 237   /* has active sessions */
 238   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 239                               &value0))
 240     {
 241       /* or is static mappings */
 242       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 243         return 0;
 244     }
 245   else
 246     return 0;
 247
 248   if (sm->forwarding_enabled)
 249     return 1;
 250
 251   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 252                                  rx_fib_index0);
 253 }
 254
 255 static inline int
 256 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 257                                   u32 proto0, u16 src_port, u16 dst_port,
 258                                   u32 thread_index, u32 sw_if_index)
 259 {
 260   snat_session_key_t key0;
 261   clib_bihash_kv_8_8_t kv0, value0;
 262   snat_interface_t *i;
 263
 264   /* src NAT check */
 265   key0.addr = ip0->src_address;
 266   key0.port = src_port;
 267   key0.protocol = proto0;
 268   key0.fib_index = sm->outside_fib_index;
 269   kv0.key = key0.as_u64;
 270
 271   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 272                                &value0))
 273     return 1;
 274
 275   /* dst NAT check */
 276   key0.addr = ip0->dst_address;
 277   key0.port = dst_port;
 278   key0.protocol = proto0;
 279   key0.fib_index = sm->inside_fib_index;
 280   kv0.key = key0.as_u64;
 281   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 282                                &value0))
 283   {
 284     /* hairpinning */
 285     pool_foreach (i, sm->output_feature_interfaces,
 286     ({
 287       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 288         return 0;
 289     }));
 290     return 1;
 291   }
 292
 293   return 0;
 294 }
 295
 296 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 297                       ip4_header_t * ip0,
 298                       u32 rx_fib_index0,
 299                       snat_session_key_t * key0,
 300                       snat_session_t ** sessionp,
 301                       vlib_node_runtime_t * node,
 302                       u32 next0,
 303                       u32 thread_index)
 304 {
 305   snat_user_t *u;
 306   snat_session_t *s;
 307   clib_bihash_kv_8_8_t kv0;
 308   snat_session_key_t key1;
 309   u32 address_index = ~0;
 310   u32 outside_fib_index;
 311   uword * p;
 312   udp_header_t * udp0 = ip4_next_header (ip0);
 313   u8 is_sm = 0;
 314
 315   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 316     {
 317       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 318       nat_ipfix_logging_max_sessions(sm->max_translations);
 319       return SNAT_IN2OUT_NEXT_DROP;
 320     }
 321
 322   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 323   if (! p)
 324     {
 325       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 326       return SNAT_IN2OUT_NEXT_DROP;
 327     }
 328   outside_fib_index = p[0];
 329
 330   key1.protocol = key0->protocol;
 331
 332   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 333                               thread_index);
 334   if (!u)
 335     {
 336       clib_warning ("create NAT user failed");
 337       return SNAT_IN2OUT_NEXT_DROP;
 338     }
 339
 340   /* First try to match static mapping by local address and port */
 341   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
 342     {
 343       /* Try to create dynamic translation */
 344       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 345                                                thread_index, &key1,
 346                                                &address_index,
 347                                                sm->port_per_thread,
 348                                                sm->per_thread_data[thread_index].snat_thread_index))
 349         {
 350           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 351           return SNAT_IN2OUT_NEXT_DROP;
 352         }
 353     }
 354   else
 355     is_sm = 1;
 356
 357   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 358   if (!s)
 359     {
 360       clib_warning ("create NAT session failed");
 361       return SNAT_IN2OUT_NEXT_DROP;
 362     }
 363
 364   if (is_sm)
 365     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 366   user_session_increment (sm, u, is_sm);
 367   s->outside_address_index = address_index;
 368   s->in2out = *key0;
 369   s->out2in = key1;
 370   s->out2in.protocol = key0->protocol;
 371   s->out2in.fib_index = outside_fib_index;
 372   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 373   s->ext_host_port = udp0->dst_port;
 374   *sessionp = s;
 375
 376   /* Add to translation hashes */
 377   kv0.key = s->in2out.as_u64;
 378   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 379   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 380                                1 /* is_add */))
 381       clib_warning ("in2out key add failed");
 382
 383   kv0.key = s->out2in.as_u64;
 384   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 385
 386   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 387                                1 /* is_add */))
 388       clib_warning ("out2in key add failed");
 389
 390   /* log NAT event */
 391   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 392                                       s->out2in.addr.as_u32,
 393                                       s->in2out.protocol,
 394                                       s->in2out.port,
 395                                       s->out2in.port,
 396                                       s->in2out.fib_index);
 397   return next0;
 398 }
 399
 400 static_always_inline
 401 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 402                                  snat_session_key_t *p_key0)
 403 {
 404   icmp46_header_t *icmp0;
 405   snat_session_key_t key0;
 406   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 407   ip4_header_t *inner_ip0 = 0;
 408   void *l4_header = 0;
 409   icmp46_header_t *inner_icmp0;
 410
 411   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 412   echo0 = (icmp_echo_header_t *)(icmp0+1);
 413
 414   if (!icmp_is_error_message (icmp0))
 415     {
 416       key0.protocol = SNAT_PROTOCOL_ICMP;
 417       key0.addr = ip0->src_address;
 418       key0.port = echo0->identifier;
 419     }
 420   else
 421     {
 422       inner_ip0 = (ip4_header_t *)(echo0+1);
 423       l4_header = ip4_next_header (inner_ip0);
 424       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 425       key0.addr = inner_ip0->dst_address;
 426       switch (key0.protocol)
 427         {
 428         case SNAT_PROTOCOL_ICMP:
 429           inner_icmp0 = (icmp46_header_t*)l4_header;
 430           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 431           key0.port = inner_echo0->identifier;
 432           break;
 433         case SNAT_PROTOCOL_UDP:
 434         case SNAT_PROTOCOL_TCP:
 435           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 436           break;
 437         default:
 438           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 439         }
 440     }
 441   *p_key0 = key0;
 442   return -1; /* success */
 443 }
 444
 445 static_always_inline int
 446 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 447 {
 448   icmp46_header_t *icmp0;
 449   nat_ed_ses_key_t key0;
 450   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 451   ip4_header_t *inner_ip0 = 0;
 452   void *l4_header = 0;
 453   icmp46_header_t *inner_icmp0;
 454
 455   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 456   echo0 = (icmp_echo_header_t *)(icmp0+1);
 457
 458   if (!icmp_is_error_message (icmp0))
 459     {
 460       key0.proto = IP_PROTOCOL_ICMP;
 461       key0.l_addr = ip0->src_address;
 462       key0.r_addr = ip0->dst_address;
 463       key0.l_port = key0.r_port = echo0->identifier;
 464     }
 465   else
 466     {
 467       inner_ip0 = (ip4_header_t *)(echo0+1);
 468       l4_header = ip4_next_header (inner_ip0);
 469       key0.proto = inner_ip0->protocol;
 470       key0.r_addr = inner_ip0->src_address;
 471       key0.l_addr = inner_ip0->dst_address;
 472       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 473         {
 474         case SNAT_PROTOCOL_ICMP:
 475           inner_icmp0 = (icmp46_header_t*)l4_header;
 476           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 477           key0.r_port = key0.l_port = inner_echo0->identifier;
 478           break;
 479         case SNAT_PROTOCOL_UDP:
 480         case SNAT_PROTOCOL_TCP:
 481           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 482           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
 483           break;
 484         default:
 485           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 486         }
 487     }
 488   *p_key0 = key0;
 489   return 0;
 490 }
 491
 492 static inline int
 493 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 494                                       u32 thread_index)
 495 {
 496   nat_ed_ses_key_t key;
 497   clib_bihash_kv_16_8_t kv, value;
 498   udp_header_t *udp;
 499   snat_session_t *s = 0;
 500   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 501
 502   if (!sm->forwarding_enabled)
 503     return 0;
 504
 505   if (ip->protocol == IP_PROTOCOL_ICMP)
 506     {
 507       if (icmp_get_ed_key (ip, &key))
 508         return 0;
 509     }
 510   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 511     {
 512       udp = ip4_next_header(ip);
 513       key.l_addr = ip->src_address;
 514       key.r_addr = ip->dst_address;
 515       key.proto = ip->protocol;
 516       key.r_port = udp->dst_port;
 517       key.l_port = udp->src_port;
 518     }
 519   else
 520     {
 521       key.l_addr = ip->src_address;
 522       key.r_addr = ip->dst_address;
 523       key.proto = ip->protocol;
 524       key.l_port = key.r_port = 0;
 525     }
 526   key.fib_index = 0;
 527   kv.key[0] = key.as_u64[0];
 528   kv.key[1] = key.as_u64[1];
 529
 530   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 531     {
 532       s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value.value);
 533       if (is_fwd_bypass_session (s))
 534         {
 535           if (ip->protocol == IP_PROTOCOL_TCP)
 536             {
 537               tcp_header_t *tcp = ip4_next_header(ip);
 538               if (nat44_set_tcp_session_state (sm, s, tcp, thread_index))
 539                 return 1;
 540             }
 541           /* Per-user LRU list maintenance */
 542           clib_dlist_remove (tsm->list_pool, s->per_user_index);
 543           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 544                               s->per_user_index);
 545           return 1;
 546         }
 547       else
 548         return 0;
 549     }
 550
 551   return 0;
 552 }
 553
 554 /**
 555  * Get address and port values to be used for ICMP packet translation
 556  * and create session if needed
 557  *
 558  * @param[in,out] sm             NAT main
 559  * @param[in,out] node           NAT node runtime
 560  * @param[in] thread_index       thread index
 561  * @param[in,out] b0             buffer containing packet to be translated
 562  * @param[out] p_proto           protocol used for matching
 563  * @param[out] p_value           address and port after NAT translation
 564  * @param[out] p_dont_translate  if packet should not be translated
 565  * @param d                      optional parameter
 566  * @param e                      optional parameter
 567  */
 568 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 569                            u32 thread_index, vlib_buffer_t *b0,
 570                            ip4_header_t *ip0, u8 *p_proto,
 571                            snat_session_key_t *p_value,
 572                            u8 *p_dont_translate, void *d, void *e)
 573 {
 574   icmp46_header_t *icmp0;
 575   u32 sw_if_index0;
 576   u32 rx_fib_index0;
 577   snat_session_key_t key0;
 578   snat_session_t *s0 = 0;
 579   u8 dont_translate = 0;
 580   clib_bihash_kv_8_8_t kv0, value0;
 581   u32 next0 = ~0;
 582   int err;
 583
 584   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 585   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 586   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 587
 588   err = icmp_get_key (ip0, &key0);
 589   if (err != -1)
 590     {
 591       b0->error = node->errors[err];
 592       next0 = SNAT_IN2OUT_NEXT_DROP;
 593       goto out;
 594     }
 595   key0.fib_index = rx_fib_index0;
 596
 597   kv0.key = key0.as_u64;
 598
 599   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 600                               &value0))
 601     {
 602       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 603         {
 604           if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
 605               ip0, SNAT_PROTOCOL_ICMP, key0.port, key0.port, thread_index, sw_if_index0)))
 606             {
 607               dont_translate = 1;
 608               goto out;
 609             }
 610         }
 611       else
 612         {
 613           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 614               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 615             {
 616               dont_translate = 1;
 617               goto out;
 618             }
 619         }
 620
 621       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 622         {
 623           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 624           next0 = SNAT_IN2OUT_NEXT_DROP;
 625           goto out;
 626         }
 627
 628       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 629                          &s0, node, next0, thread_index);
 630
 631       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 632         goto out;
 633     }
 634   else
 635     {
 636       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 637                         icmp0->type != ICMP4_echo_reply &&
 638                         !icmp_is_error_message (icmp0)))
 639         {
 640           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 641           next0 = SNAT_IN2OUT_NEXT_DROP;
 642           goto out;
 643         }
 644
 645       if (PREDICT_FALSE (value0.value == ~0ULL))
 646         {
 647           nat_ed_ses_key_t key;
 648           clib_bihash_kv_16_8_t s_kv, s_value;
 649
 650           key.as_u64[0] = 0;
 651           key.as_u64[1] = 0;
 652           if (icmp_get_ed_key (ip0, &key))
 653             {
 654               b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 655               next0 = SNAT_IN2OUT_NEXT_DROP;
 656               goto out;
 657             }
 658           key.fib_index = rx_fib_index0;
 659           s_kv.key[0] = key.as_u64[0];
 660           s_kv.key[1] = key.as_u64[1];
 661           if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 662             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 663                                     s_value.value);
 664           else
 665            {
 666               next0 = SNAT_IN2OUT_NEXT_DROP;
 667               goto out;
 668            }
 669         }
 670       else
 671         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 672                                 value0.value);
 673     }
 674
 675 out:
 676   *p_proto = key0.protocol;
 677   if (s0)
 678     *p_value = s0->out2in;
 679   *p_dont_translate = dont_translate;
 680   if (d)
 681     *(snat_session_t**)d = s0;
 682   return next0;
 683 }
 684
 685 /**
 686  * Get address and port values to be used for ICMP packet translation
 687  *
 688  * @param[in] sm                 NAT main
 689  * @param[in,out] node           NAT node runtime
 690  * @param[in] thread_index       thread index
 691  * @param[in,out] b0             buffer containing packet to be translated
 692  * @param[out] p_proto           protocol used for matching
 693  * @param[out] p_value           address and port after NAT translation
 694  * @param[out] p_dont_translate  if packet should not be translated
 695  * @param d                      optional parameter
 696  * @param e                      optional parameter
 697  */
 698 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 699                            u32 thread_index, vlib_buffer_t *b0,
 700                            ip4_header_t *ip0, u8 *p_proto,
 701                            snat_session_key_t *p_value,
 702                            u8 *p_dont_translate, void *d, void *e)
 703 {
 704   icmp46_header_t *icmp0;
 705   u32 sw_if_index0;
 706   u32 rx_fib_index0;
 707   snat_session_key_t key0;
 708   snat_session_key_t sm0;
 709   u8 dont_translate = 0;
 710   u8 is_addr_only;
 711   u32 next0 = ~0;
 712   int err;
 713
 714   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 715   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 716   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 717
 718   err = icmp_get_key (ip0, &key0);
 719   if (err != -1)
 720     {
 721       b0->error = node->errors[err];
 722       next0 = SNAT_IN2OUT_NEXT_DROP;
 723       goto out2;
 724     }
 725   key0.fib_index = rx_fib_index0;
 726
 727   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
 728     {
 729       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 730           IP_PROTOCOL_ICMP, rx_fib_index0)))
 731         {
 732           dont_translate = 1;
 733           goto out;
 734         }
 735
 736       if (icmp_is_error_message (icmp0))
 737         {
 738           next0 = SNAT_IN2OUT_NEXT_DROP;
 739           goto out;
 740         }
 741
 742       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 743       next0 = SNAT_IN2OUT_NEXT_DROP;
 744       goto out;
 745     }
 746
 747   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 748                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 749                     !icmp_is_error_message (icmp0)))
 750     {
 751       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 752       next0 = SNAT_IN2OUT_NEXT_DROP;
 753       goto out;
 754     }
 755
 756 out:
 757   *p_value = sm0;
 758 out2:
 759   *p_proto = key0.protocol;
 760   *p_dont_translate = dont_translate;
 761   return next0;
 762 }
 763
 764 static inline u32 icmp_in2out (snat_main_t *sm,
 765                                vlib_buffer_t * b0,
 766                                ip4_header_t * ip0,
 767                                icmp46_header_t * icmp0,
 768                                u32 sw_if_index0,
 769                                u32 rx_fib_index0,
 770                                vlib_node_runtime_t * node,
 771                                u32 next0,
 772                                u32 thread_index,
 773                                void *d,
 774                                void *e)
 775 {
 776   snat_session_key_t sm0;
 777   u8 protocol;
 778   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 779   ip4_header_t *inner_ip0;
 780   void *l4_header = 0;
 781   icmp46_header_t *inner_icmp0;
 782   u8 dont_translate;
 783   u32 new_addr0, old_addr0;
 784   u16 old_id0, new_id0;
 785   ip_csum_t sum0;
 786   u16 checksum0;
 787   u32 next0_tmp;
 788
 789   echo0 = (icmp_echo_header_t *)(icmp0+1);
 790
 791   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 792                                        &protocol, &sm0, &dont_translate, d, e);
 793   if (next0_tmp != ~0)
 794     next0 = next0_tmp;
 795   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 796     goto out;
 797
 798   sum0 = ip_incremental_checksum (0, icmp0,
 799                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 800   checksum0 = ~ip_csum_fold (sum0);
 801   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 802     {
 803       next0 = SNAT_IN2OUT_NEXT_DROP;
 804       goto out;
 805     }
 806
 807   old_addr0 = ip0->src_address.as_u32;
 808   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 809   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 810     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 811
 812   sum0 = ip0->checksum;
 813   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 814                          src_address /* changed member */);
 815   ip0->checksum = ip_csum_fold (sum0);
 816
 817   if (icmp0->checksum == 0)
 818     icmp0->checksum = 0xffff;
 819
 820   if (!icmp_is_error_message (icmp0))
 821     {
 822       new_id0 = sm0.port;
 823       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 824         {
 825           old_id0 = echo0->identifier;
 826           new_id0 = sm0.port;
 827           echo0->identifier = new_id0;
 828
 829           sum0 = icmp0->checksum;
 830           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 831                                  identifier);
 832           icmp0->checksum = ip_csum_fold (sum0);
 833         }
 834     }
 835   else
 836     {
 837       inner_ip0 = (ip4_header_t *)(echo0+1);
 838       l4_header = ip4_next_header (inner_ip0);
 839
 840       if (!ip4_header_checksum_is_valid (inner_ip0))
 841         {
 842           next0 = SNAT_IN2OUT_NEXT_DROP;
 843           goto out;
 844         }
 845
 846       old_addr0 = inner_ip0->dst_address.as_u32;
 847       inner_ip0->dst_address = sm0.addr;
 848       new_addr0 = inner_ip0->dst_address.as_u32;
 849
 850       sum0 = icmp0->checksum;
 851       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 852                              dst_address /* changed member */);
 853       icmp0->checksum = ip_csum_fold (sum0);
 854
 855       switch (protocol)
 856         {
 857           case SNAT_PROTOCOL_ICMP:
 858             inner_icmp0 = (icmp46_header_t*)l4_header;
 859             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 860
 861             old_id0 = inner_echo0->identifier;
 862             new_id0 = sm0.port;
 863             inner_echo0->identifier = new_id0;
 864
 865             sum0 = icmp0->checksum;
 866             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 867                                    identifier);
 868             icmp0->checksum = ip_csum_fold (sum0);
 869             break;
 870           case SNAT_PROTOCOL_UDP:
 871           case SNAT_PROTOCOL_TCP:
 872             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 873             new_id0 = sm0.port;
 874             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 875
 876             sum0 = icmp0->checksum;
 877             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 878                                    dst_port);
 879             icmp0->checksum = ip_csum_fold (sum0);
 880             break;
 881           default:
 882             ASSERT(0);
 883         }
 884     }
 885
 886 out:
 887   return next0;
 888 }
 889
 890 /**
 891  * @brief Hairpinning
 892  *
 893  * Hairpinning allows two endpoints on the internal side of the NAT to
 894  * communicate even if they only use each other's external IP addresses
 895  * and ports.
 896  *
 897  * @param sm     NAT main.
 898  * @param b0     Vlib buffer.
 899  * @param ip0    IP header.
 900  * @param udp0   UDP header.
 901  * @param tcp0   TCP header.
 902  * @param proto0 NAT protocol.
 903  */
 904 static inline int
 905 snat_hairpinning (snat_main_t *sm,
 906                   vlib_buffer_t * b0,
 907                   ip4_header_t * ip0,
 908                   udp_header_t * udp0,
 909                   tcp_header_t * tcp0,
 910                   u32 proto0)
 911 {
 912   snat_session_key_t key0, sm0;
 913   snat_session_t * s0;
 914   clib_bihash_kv_8_8_t kv0, value0;
 915   ip_csum_t sum0;
 916   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 917   u16 new_dst_port0, old_dst_port0;
 918
 919   key0.addr = ip0->dst_address;
 920   key0.port = udp0->dst_port;
 921   key0.protocol = proto0;
 922   key0.fib_index = sm->outside_fib_index;
 923   kv0.key = key0.as_u64;
 924
 925   /* Check if destination is static mappings */
 926   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 927     {
 928       new_dst_addr0 = sm0.addr.as_u32;
 929       new_dst_port0 = sm0.port;
 930       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 931     }
 932   /* or active session */
 933   else
 934     {
 935       if (sm->num_workers > 1)
 936         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 937       else
 938         ti = sm->num_workers;
 939
 940       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 941         {
 942           si = value0.value;
 943
 944           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 945           new_dst_addr0 = s0->in2out.addr.as_u32;
 946           new_dst_port0 = s0->in2out.port;
 947           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 948         }
 949     }
 950
 951   /* Destination is behind the same NAT, use internal address and port */
 952   if (new_dst_addr0)
 953     {
 954       old_dst_addr0 = ip0->dst_address.as_u32;
 955       ip0->dst_address.as_u32 = new_dst_addr0;
 956       sum0 = ip0->checksum;
 957       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 958                              ip4_header_t, dst_address);
 959       ip0->checksum = ip_csum_fold (sum0);
 960
 961       old_dst_port0 = tcp0->dst;
 962       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 963         {
 964           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 965             {
 966               tcp0->dst = new_dst_port0;
 967               sum0 = tcp0->checksum;
 968               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 969                                      ip4_header_t, dst_address);
 970               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 971                                      ip4_header_t /* cheat */, length);
 972               tcp0->checksum = ip_csum_fold(sum0);
 973             }
 974           else
 975             {
 976               udp0->dst_port = new_dst_port0;
 977               udp0->checksum = 0;
 978             }
 979         }
 980       else
 981         {
 982           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 983             {
 984               sum0 = tcp0->checksum;
 985               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 986                                      ip4_header_t, dst_address);
 987               tcp0->checksum = ip_csum_fold(sum0);
 988             }
 989         }
 990       return 1;
 991     }
 992   return 0;
 993 }
 994
 995 static inline void
 996 snat_icmp_hairpinning (snat_main_t *sm,
 997                        vlib_buffer_t * b0,
 998                        ip4_header_t * ip0,
 999                        icmp46_header_t * icmp0)
1000 {
1001   snat_session_key_t key0, sm0;
1002   clib_bihash_kv_8_8_t kv0, value0;
1003   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
1004   ip_csum_t sum0;
1005   snat_session_t *s0;
1006
1007   if (!icmp_is_error_message (icmp0))
1008     {
1009       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
1010       u16 icmp_id0 = echo0->identifier;
1011       key0.addr = ip0->dst_address;
1012       key0.port = icmp_id0;
1013       key0.protocol = SNAT_PROTOCOL_ICMP;
1014       key0.fib_index = sm->outside_fib_index;
1015       kv0.key = key0.as_u64;
1016
1017       if (sm->num_workers > 1)
1018         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
1019       else
1020         ti = sm->num_workers;
1021
1022       /* Check if destination is in active sessions */
1023       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1024                                   &value0))
1025         {
1026           /* or static mappings */
1027           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1028             {
1029               new_dst_addr0 = sm0.addr.as_u32;
1030               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1031             }
1032         }
1033       else
1034         {
1035           si = value0.value;
1036
1037           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1038           new_dst_addr0 = s0->in2out.addr.as_u32;
1039           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1040           echo0->identifier = s0->in2out.port;
1041           sum0 = icmp0->checksum;
1042           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1043                                  icmp_echo_header_t, identifier);
1044           icmp0->checksum = ip_csum_fold (sum0);
1045         }
1046
1047       /* Destination is behind the same NAT, use internal address and port */
1048       if (new_dst_addr0)
1049         {
1050           old_dst_addr0 = ip0->dst_address.as_u32;
1051           ip0->dst_address.as_u32 = new_dst_addr0;
1052           sum0 = ip0->checksum;
1053           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1054                                  ip4_header_t, dst_address);
1055           ip0->checksum = ip_csum_fold (sum0);
1056         }
1057     }
1058
1059 }
1060
1061 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1062                                          vlib_buffer_t * b0,
1063                                          ip4_header_t * ip0,
1064                                          icmp46_header_t * icmp0,
1065                                          u32 sw_if_index0,
1066                                          u32 rx_fib_index0,
1067                                          vlib_node_runtime_t * node,
1068                                          u32 next0,
1069                                          f64 now,
1070                                          u32 thread_index,
1071                                          snat_session_t ** p_s0)
1072 {
1073   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1074                       next0, thread_index, p_s0, 0);
1075   snat_session_t * s0 = *p_s0;
1076   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1077     {
1078       /* Hairpinning */
1079       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1080         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1081       /* Accounting */
1082       s0->last_heard = now;
1083       s0->total_pkts++;
1084       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1085       /* Per-user LRU list maintenance */
1086       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1087                          s0->per_user_index);
1088       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1089                           s0->per_user_list_head_index,
1090                           s0->per_user_index);
1091     }
1092   return next0;
1093 }
1094 static inline void
1095 snat_hairpinning_unknown_proto (snat_main_t *sm,
1096                                 vlib_buffer_t * b,
1097                                 ip4_header_t * ip)
1098 {
1099   u32 old_addr, new_addr = 0, ti = 0;
1100   clib_bihash_kv_8_8_t kv, value;
1101   clib_bihash_kv_16_8_t s_kv, s_value;
1102   nat_ed_ses_key_t key;
1103   snat_session_key_t m_key;
1104   snat_static_mapping_t *m;
1105   ip_csum_t sum;
1106   snat_session_t *s;
1107
1108   old_addr = ip->dst_address.as_u32;
1109   key.l_addr.as_u32 = ip->dst_address.as_u32;
1110   key.r_addr.as_u32 = ip->src_address.as_u32;
1111   key.fib_index = sm->outside_fib_index;
1112   key.proto = ip->protocol;
1113   key.r_port = 0;
1114   key.l_port = 0;
1115   s_kv.key[0] = key.as_u64[0];
1116   s_kv.key[1] = key.as_u64[1];
1117   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1118     {
1119       m_key.addr = ip->dst_address;
1120       m_key.fib_index = sm->outside_fib_index;
1121       m_key.port = 0;
1122       m_key.protocol = 0;
1123       kv.key = m_key.as_u64;
1124       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1125         return;
1126
1127       m = pool_elt_at_index (sm->static_mappings, value.value);
1128       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1129         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1130       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1131     }
1132   else
1133     {
1134       if (sm->num_workers > 1)
1135         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1136       else
1137         ti = sm->num_workers;
1138
1139       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1140       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1141         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1142       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1143     }
1144   sum = ip->checksum;
1145   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1146   ip->checksum = ip_csum_fold (sum);
1147 }
1148
1149 static snat_session_t *
1150 snat_in2out_unknown_proto (snat_main_t *sm,
1151                            vlib_buffer_t * b,
1152                            ip4_header_t * ip,
1153                            u32 rx_fib_index,
1154                            u32 thread_index,
1155                            f64 now,
1156                            vlib_main_t * vm,
1157                            vlib_node_runtime_t * node)
1158 {
1159   clib_bihash_kv_8_8_t kv, value;
1160   clib_bihash_kv_16_8_t s_kv, s_value;
1161   snat_static_mapping_t *m;
1162   snat_session_key_t m_key;
1163   u32 old_addr, new_addr = 0;
1164   ip_csum_t sum;
1165   snat_user_t *u;
1166   dlist_elt_t *head, *elt;
1167   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1168   u32 elt_index, head_index, ses_index;
1169   snat_session_t * s;
1170   nat_ed_ses_key_t key;
1171   u32 address_index = ~0;
1172   int i;
1173   u8 is_sm = 0;
1174
1175   old_addr = ip->src_address.as_u32;
1176
1177   key.l_addr = ip->src_address;
1178   key.r_addr = ip->dst_address;
1179   key.fib_index = rx_fib_index;
1180   key.proto = ip->protocol;
1181   key.l_port = 0;
1182   key.r_port = 0;
1183   s_kv.key[0] = key.as_u64[0];
1184   s_kv.key[1] = key.as_u64[1];
1185
1186   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1187     {
1188       s = pool_elt_at_index (tsm->sessions, s_value.value);
1189       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1190     }
1191   else
1192     {
1193       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1194         {
1195           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1196           nat_ipfix_logging_max_sessions(sm->max_translations);
1197           return 0;
1198         }
1199
1200       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1201                                   thread_index);
1202       if (!u)
1203         {
1204           clib_warning ("create NAT user failed");
1205           return 0;
1206         }
1207
1208       m_key.addr = ip->src_address;
1209       m_key.port = 0;
1210       m_key.protocol = 0;
1211       m_key.fib_index = rx_fib_index;
1212       kv.key = m_key.as_u64;
1213
1214       /* Try to find static mapping first */
1215       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1216         {
1217           m = pool_elt_at_index (sm->static_mappings, value.value);
1218           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1219           is_sm = 1;
1220           goto create_ses;
1221         }
1222       /* Fallback to 3-tuple key */
1223       else
1224         {
1225           /* Choose same out address as for TCP/UDP session to same destination */
1226           head_index = u->sessions_per_user_list_head_index;
1227           head = pool_elt_at_index (tsm->list_pool, head_index);
1228           elt_index = head->next;
1229           if (PREDICT_FALSE (elt_index == ~0))
1230             ses_index = ~0;
1231           else
1232             {
1233               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1234               ses_index = elt->value;
1235             }
1236
1237           while (ses_index != ~0)
1238             {
1239               s =  pool_elt_at_index (tsm->sessions, ses_index);
1240               elt_index = elt->next;
1241               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1242               ses_index = elt->value;
1243
1244               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1245                 {
1246                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1247                   address_index = s->outside_address_index;
1248
1249                   key.fib_index = sm->outside_fib_index;
1250                   key.l_addr.as_u32 = new_addr;
1251                   s_kv.key[0] = key.as_u64[0];
1252                   s_kv.key[1] = key.as_u64[1];
1253                   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1254                     break;
1255
1256                   goto create_ses;
1257                 }
1258             }
1259           key.fib_index = sm->outside_fib_index;
1260           for (i = 0; i < vec_len (sm->addresses); i++)
1261             {
1262               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1263               s_kv.key[0] = key.as_u64[0];
1264               s_kv.key[1] = key.as_u64[1];
1265               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1266                 {
1267                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1268                   address_index = i;
1269                   goto create_ses;
1270                 }
1271             }
1272           return 0;
1273         }
1274
1275 create_ses:
1276       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1277       if (!s)
1278         {
1279           clib_warning ("create NAT session failed");
1280           return 0;
1281         }
1282
1283       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1284       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1285       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1286       s->outside_address_index = address_index;
1287       s->out2in.addr.as_u32 = new_addr;
1288       s->out2in.fib_index = sm->outside_fib_index;
1289       s->in2out.addr.as_u32 = old_addr;
1290       s->in2out.fib_index = rx_fib_index;
1291       s->in2out.port = s->out2in.port = ip->protocol;
1292       if (is_sm)
1293         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1294       user_session_increment (sm, u, is_sm);
1295
1296       /* Add to lookup tables */
1297       key.l_addr.as_u32 = old_addr;
1298       key.r_addr = ip->dst_address;
1299       key.proto = ip->protocol;
1300       key.fib_index = rx_fib_index;
1301       s_kv.key[0] = key.as_u64[0];
1302       s_kv.key[1] = key.as_u64[1];
1303       s_kv.value = s - tsm->sessions;
1304       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1305         clib_warning ("in2out key add failed");
1306
1307       key.l_addr.as_u32 = new_addr;
1308       key.fib_index = sm->outside_fib_index;
1309       s_kv.key[0] = key.as_u64[0];
1310       s_kv.key[1] = key.as_u64[1];
1311       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1312         clib_warning ("out2in key add failed");
1313   }
1314
1315   /* Update IP checksum */
1316   sum = ip->checksum;
1317   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1318   ip->checksum = ip_csum_fold (sum);
1319
1320   /* Accounting */
1321   s->last_heard = now;
1322   s->total_pkts++;
1323   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1324   /* Per-user LRU list maintenance */
1325   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1326   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1327                       s->per_user_index);
1328
1329   /* Hairpinning */
1330   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1331     snat_hairpinning_unknown_proto(sm, b, ip);
1332
1333   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1334     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1335
1336   return s;
1337 }
1338
1339 static snat_session_t *
1340 snat_in2out_lb (snat_main_t *sm,
1341                 vlib_buffer_t * b,
1342                 ip4_header_t * ip,
1343                 u32 rx_fib_index,
1344                 u32 thread_index,
1345                 f64 now,
1346                 vlib_main_t * vm,
1347                 vlib_node_runtime_t * node)
1348 {
1349   nat_ed_ses_key_t key;
1350   clib_bihash_kv_16_8_t s_kv, s_value;
1351   udp_header_t *udp = ip4_next_header (ip);
1352   tcp_header_t *tcp = (tcp_header_t *) udp;
1353   snat_session_t *s = 0;
1354   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1355   u32 old_addr, new_addr;
1356   u16 new_port, old_port;
1357   ip_csum_t sum;
1358   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1359   snat_session_key_t e_key, l_key;
1360   snat_user_t *u;
1361   u8 lb;
1362
1363   old_addr = ip->src_address.as_u32;
1364
1365   key.l_addr = ip->src_address;
1366   key.r_addr = ip->dst_address;
1367   key.fib_index = rx_fib_index;
1368   key.proto = ip->protocol;
1369   key.r_port = udp->dst_port;
1370   key.l_port = udp->src_port;
1371   s_kv.key[0] = key.as_u64[0];
1372   s_kv.key[1] = key.as_u64[1];
1373
1374   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1375     {
1376       s = pool_elt_at_index (tsm->sessions, s_value.value);
1377       if (is_fwd_bypass_session (s))
1378         {
1379           if (ip->protocol == IP_PROTOCOL_TCP)
1380             {
1381               if (nat44_set_tcp_session_state (sm, s, tcp, thread_index))
1382                 return 0;
1383             }
1384           /* Per-user LRU list maintenance */
1385           clib_dlist_remove (tsm->list_pool, s->per_user_index);
1386           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1387                               s->per_user_index);
1388           return 0;
1389         }
1390     }
1391   else
1392     {
1393       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1394         {
1395           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1396           nat_ipfix_logging_max_sessions(sm->max_translations);
1397           return 0;
1398         }
1399
1400       l_key.addr = ip->src_address;
1401       l_key.port = udp->src_port;
1402       l_key.protocol = proto;
1403       l_key.fib_index = rx_fib_index;
1404       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0, &lb))
1405         return 0;
1406
1407       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1408                                   thread_index);
1409       if (!u)
1410         {
1411           clib_warning ("create NAT user failed");
1412           return 0;
1413         }
1414
1415       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1416       if (!s)
1417         {
1418           clib_warning ("create NAT session failed");
1419           return 0;
1420         }
1421
1422       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1423       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1424       if (lb)
1425         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1426       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1427       s->outside_address_index = ~0;
1428       s->in2out = l_key;
1429       s->out2in = e_key;
1430       s->out2in.protocol = l_key.protocol;
1431       user_session_increment (sm, u, 1 /* static */);
1432
1433       /* Add to lookup tables */
1434       s_kv.value = s - tsm->sessions;
1435       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1436         clib_warning ("in2out-ed key add failed");
1437
1438       key.l_addr = e_key.addr;
1439       key.fib_index = e_key.fib_index;
1440       key.l_port = e_key.port;
1441       s_kv.key[0] = key.as_u64[0];
1442       s_kv.key[1] = key.as_u64[1];
1443       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1444         clib_warning ("out2in-ed key add failed");
1445     }
1446
1447   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1448
1449   /* Update IP checksum */
1450   sum = ip->checksum;
1451   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1452   if (is_twice_nat_session (s))
1453     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1454                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1455   ip->checksum = ip_csum_fold (sum);
1456
1457   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1458     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1459
1460   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1461     {
1462       old_port = tcp->src_port;
1463       tcp->src_port = s->out2in.port;
1464       new_port = tcp->src_port;
1465
1466       sum = tcp->checksum;
1467       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1468       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1469       if (is_twice_nat_session (s))
1470         {
1471           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1472                                 s->ext_host_addr.as_u32, ip4_header_t,
1473                                 dst_address);
1474           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1475                                 ip4_header_t, length);
1476           tcp->dst_port = s->ext_host_port;
1477           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1478         }
1479       tcp->checksum = ip_csum_fold(sum);
1480       if (nat44_set_tcp_session_state (sm, s, tcp, thread_index))
1481         return s;
1482     }
1483   else
1484     {
1485       udp->src_port = s->out2in.port;
1486       if (is_twice_nat_session (s))
1487         {
1488           udp->dst_port = s->ext_host_port;
1489           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1490         }
1491       udp->checksum = 0;
1492     }
1493
1494   /* Accounting */
1495   s->last_heard = now;
1496   s->total_pkts++;
1497   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1498   /* Per-user LRU list maintenance */
1499   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1500   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1501                       s->per_user_index);
1502   return s;
1503 }
1504
1505 static inline uword
1506 snat_in2out_node_fn_inline (vlib_main_t * vm,
1507                             vlib_node_runtime_t * node,
1508                             vlib_frame_t * frame, int is_slow_path,
1509                             int is_output_feature)
1510 {
1511   u32 n_left_from, * from, * to_next;
1512   snat_in2out_next_t next_index;
1513   u32 pkts_processed = 0;
1514   snat_main_t * sm = &snat_main;
1515   f64 now = vlib_time_now (vm);
1516   u32 stats_node_index;
1517   u32 thread_index = vlib_get_thread_index ();
1518
1519   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1520     snat_in2out_node.index;
1521
1522   from = vlib_frame_vector_args (frame);
1523   n_left_from = frame->n_vectors;
1524   next_index = node->cached_next_index;
1525
1526   while (n_left_from > 0)
1527     {
1528       u32 n_left_to_next;
1529
1530       vlib_get_next_frame (vm, node, next_index,
1531                            to_next, n_left_to_next);
1532
1533       while (n_left_from >= 4 && n_left_to_next >= 2)
1534         {
1535           u32 bi0, bi1;
1536           vlib_buffer_t * b0, * b1;
1537           u32 next0, next1;
1538           u32 sw_if_index0, sw_if_index1;
1539           ip4_header_t * ip0, * ip1;
1540           ip_csum_t sum0, sum1;
1541           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1542           u16 old_port0, new_port0, old_port1, new_port1;
1543           udp_header_t * udp0, * udp1;
1544           tcp_header_t * tcp0, * tcp1;
1545           icmp46_header_t * icmp0, * icmp1;
1546           snat_session_key_t key0, key1;
1547           u32 rx_fib_index0, rx_fib_index1;
1548           u32 proto0, proto1;
1549           snat_session_t * s0 = 0, * s1 = 0;
1550           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1551           u32 iph_offset0 = 0, iph_offset1 = 0;
1552
1553           /* Prefetch next iteration. */
1554           {
1555             vlib_buffer_t * p2, * p3;
1556
1557             p2 = vlib_get_buffer (vm, from[2]);
1558             p3 = vlib_get_buffer (vm, from[3]);
1559
1560             vlib_prefetch_buffer_header (p2, LOAD);
1561             vlib_prefetch_buffer_header (p3, LOAD);
1562
1563             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1564             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1565           }
1566
1567           /* speculatively enqueue b0 and b1 to the current next frame */
1568           to_next[0] = bi0 = from[0];
1569           to_next[1] = bi1 = from[1];
1570           from += 2;
1571           to_next += 2;
1572           n_left_from -= 2;
1573           n_left_to_next -= 2;
1574
1575           b0 = vlib_get_buffer (vm, bi0);
1576           b1 = vlib_get_buffer (vm, bi1);
1577
1578           if (is_output_feature)
1579             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1580
1581           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1582                  iph_offset0);
1583
1584           udp0 = ip4_next_header (ip0);
1585           tcp0 = (tcp_header_t *) udp0;
1586           icmp0 = (icmp46_header_t *) udp0;
1587
1588           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1589           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1590                                    sw_if_index0);
1591
1592           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1593
1594           if (PREDICT_FALSE(ip0->ttl == 1))
1595             {
1596               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1597               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1598                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1599                                            0);
1600               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1601               goto trace00;
1602             }
1603
1604           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1605
1606           /* Next configured feature, probably ip4-lookup */
1607           if (is_slow_path)
1608             {
1609               if (PREDICT_FALSE (proto0 == ~0))
1610                 {
1611                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1612                                                   thread_index, now, vm, node);
1613                   if (!s0)
1614                     next0 = SNAT_IN2OUT_NEXT_DROP;
1615                   goto trace00;
1616                 }
1617
1618               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1619                 {
1620                   next0 = icmp_in2out_slow_path
1621                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1622                      node, next0, now, thread_index, &s0);
1623                   goto trace00;
1624                 }
1625             }
1626           else
1627             {
1628               if (is_output_feature)
1629                 {
1630                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
1631                     goto trace00;
1632                 }
1633
1634               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1635                 {
1636                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1637                   goto trace00;
1638                 }
1639
1640               if (ip4_is_fragment (ip0))
1641                 {
1642                   next0 = SNAT_IN2OUT_NEXT_REASS;
1643                   goto trace00;
1644                 }
1645             }
1646
1647           key0.addr = ip0->src_address;
1648           key0.port = udp0->src_port;
1649           key0.protocol = proto0;
1650           key0.fib_index = rx_fib_index0;
1651
1652           kv0.key = key0.as_u64;
1653
1654           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1655               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1656             {
1657               if (is_slow_path)
1658                 {
1659                   if (is_output_feature)
1660                     {
1661                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1662                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1663                         goto trace00;
1664                     }
1665                   else
1666                     {
1667                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1668                           ip0, proto0, rx_fib_index0, thread_index)))
1669                         goto trace00;
1670                     }
1671
1672                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1673                                      &s0, node, next0, thread_index);
1674                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1675                     goto trace00;
1676                 }
1677               else
1678                 {
1679                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1680                   goto trace00;
1681                 }
1682             }
1683           else
1684             {
1685               if (PREDICT_FALSE (value0.value == ~0ULL))
1686                 {
1687                   if (is_slow_path)
1688                     {
1689                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1690                                           thread_index, now, vm, node);
1691                       if (!s0 && !sm->forwarding_enabled)
1692                         next0 = SNAT_IN2OUT_NEXT_DROP;
1693                       goto trace00;
1694                     }
1695                   else
1696                     {
1697                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1698                       goto trace00;
1699                     }
1700                 }
1701               else
1702                 {
1703                   s0 = pool_elt_at_index (
1704                     sm->per_thread_data[thread_index].sessions,
1705                     value0.value);
1706                 }
1707             }
1708
1709           b0->flags |= VNET_BUFFER_F_IS_NATED;
1710
1711           old_addr0 = ip0->src_address.as_u32;
1712           ip0->src_address = s0->out2in.addr;
1713           new_addr0 = ip0->src_address.as_u32;
1714           if (!is_output_feature)
1715             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1716
1717           sum0 = ip0->checksum;
1718           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1719                                  ip4_header_t,
1720                                  src_address /* changed member */);
1721           ip0->checksum = ip_csum_fold (sum0);
1722
1723           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1724             {
1725               old_port0 = tcp0->src_port;
1726               tcp0->src_port = s0->out2in.port;
1727               new_port0 = tcp0->src_port;
1728
1729               sum0 = tcp0->checksum;
1730               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1731                                      ip4_header_t,
1732                                      dst_address /* changed member */);
1733               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1734                                      ip4_header_t /* cheat */,
1735                                      length /* changed member */);
1736               tcp0->checksum = ip_csum_fold(sum0);
1737               if (nat44_set_tcp_session_state (sm, s0, tcp0, thread_index))
1738                 goto trace00;
1739             }
1740           else
1741             {
1742               old_port0 = udp0->src_port;
1743               udp0->src_port = s0->out2in.port;
1744               udp0->checksum = 0;
1745             }
1746
1747           /* Accounting */
1748           s0->last_heard = now;
1749           s0->total_pkts++;
1750           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1751           /* Per-user LRU list maintenance */
1752           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1753                              s0->per_user_index);
1754           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1755                               s0->per_user_list_head_index,
1756                               s0->per_user_index);
1757         trace00:
1758
1759           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1760                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1761             {
1762               snat_in2out_trace_t *t =
1763                  vlib_add_trace (vm, node, b0, sizeof (*t));
1764               t->is_slow_path = is_slow_path;
1765               t->sw_if_index = sw_if_index0;
1766               t->next_index = next0;
1767                   t->session_index = ~0;
1768               if (s0)
1769                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1770             }
1771
1772           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1773
1774           if (is_output_feature)
1775             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1776
1777           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1778                  iph_offset1);
1779
1780           udp1 = ip4_next_header (ip1);
1781           tcp1 = (tcp_header_t *) udp1;
1782           icmp1 = (icmp46_header_t *) udp1;
1783
1784           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1785           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1786                                    sw_if_index1);
1787
1788           if (PREDICT_FALSE(ip1->ttl == 1))
1789             {
1790               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1791               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1792                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1793                                            0);
1794               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1795               goto trace01;
1796             }
1797
1798           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1799
1800           /* Next configured feature, probably ip4-lookup */
1801           if (is_slow_path)
1802             {
1803               if (PREDICT_FALSE (proto1 == ~0))
1804                 {
1805                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1806                                                   thread_index, now, vm, node);
1807                   if (!s1)
1808                     next1 = SNAT_IN2OUT_NEXT_DROP;
1809                   goto trace01;
1810                 }
1811
1812               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1813                 {
1814                   next1 = icmp_in2out_slow_path
1815                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1816                      next1, now, thread_index, &s1);
1817                   goto trace01;
1818                 }
1819             }
1820           else
1821             {
1822               if (is_output_feature)
1823                 {
1824                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip1, thread_index)))
1825                     goto trace01;
1826                 }
1827
1828               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1829                 {
1830                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1831                   goto trace01;
1832                 }
1833
1834               if (ip4_is_fragment (ip1))
1835                 {
1836                   next1 = SNAT_IN2OUT_NEXT_REASS;
1837                   goto trace01;
1838                 }
1839             }
1840
1841           key1.addr = ip1->src_address;
1842           key1.port = udp1->src_port;
1843           key1.protocol = proto1;
1844           key1.fib_index = rx_fib_index1;
1845
1846           kv1.key = key1.as_u64;
1847
1848             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1849                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1850             {
1851               if (is_slow_path)
1852                 {
1853                   if (is_output_feature)
1854                     {
1855                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1856                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1857                         goto trace01;
1858                     }
1859                   else
1860                     {
1861                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1862                           ip1, proto1, rx_fib_index1, thread_index)))
1863                         goto trace01;
1864                     }
1865
1866                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1867                                      &s1, node, next1, thread_index);
1868                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1869                     goto trace01;
1870                 }
1871               else
1872                 {
1873                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1874                   goto trace01;
1875                 }
1876             }
1877           else
1878             {
1879               if (PREDICT_FALSE (value1.value == ~0ULL))
1880                 {
1881                   if (is_slow_path)
1882                     {
1883                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1884                                           thread_index, now, vm, node);
1885                       if (!s1 && !sm->forwarding_enabled)
1886                         next1 = SNAT_IN2OUT_NEXT_DROP;
1887                       goto trace01;
1888                     }
1889                   else
1890                     {
1891                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1892                       goto trace01;
1893                     }
1894                 }
1895               else
1896                 {
1897                   s1 = pool_elt_at_index (
1898                     sm->per_thread_data[thread_index].sessions,
1899                     value1.value);
1900                 }
1901             }
1902
1903           b1->flags |= VNET_BUFFER_F_IS_NATED;
1904
1905           old_addr1 = ip1->src_address.as_u32;
1906           ip1->src_address = s1->out2in.addr;
1907           new_addr1 = ip1->src_address.as_u32;
1908           if (!is_output_feature)
1909             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1910
1911           sum1 = ip1->checksum;
1912           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1913                                  ip4_header_t,
1914                                  src_address /* changed member */);
1915           ip1->checksum = ip_csum_fold (sum1);
1916
1917           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1918             {
1919               old_port1 = tcp1->src_port;
1920               tcp1->src_port = s1->out2in.port;
1921               new_port1 = tcp1->src_port;
1922
1923               sum1 = tcp1->checksum;
1924               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1925                                      ip4_header_t,
1926                                      dst_address /* changed member */);
1927               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1928                                      ip4_header_t /* cheat */,
1929                                      length /* changed member */);
1930               tcp1->checksum = ip_csum_fold(sum1);
1931               if (nat44_set_tcp_session_state (sm, s1, tcp1, thread_index))
1932                 goto trace01;
1933             }
1934           else
1935             {
1936               old_port1 = udp1->src_port;
1937               udp1->src_port = s1->out2in.port;
1938               udp1->checksum = 0;
1939             }
1940
1941           /* Accounting */
1942           s1->last_heard = now;
1943           s1->total_pkts++;
1944           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1945           /* Per-user LRU list maintenance */
1946           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1947                              s1->per_user_index);
1948           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1949                               s1->per_user_list_head_index,
1950                               s1->per_user_index);
1951         trace01:
1952
1953           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1954                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1955             {
1956               snat_in2out_trace_t *t =
1957                  vlib_add_trace (vm, node, b1, sizeof (*t));
1958               t->sw_if_index = sw_if_index1;
1959               t->next_index = next1;
1960               t->session_index = ~0;
1961               if (s1)
1962                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1963             }
1964
1965           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1966
1967           /* verify speculative enqueues, maybe switch current next frame */
1968           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1969                                            to_next, n_left_to_next,
1970                                            bi0, bi1, next0, next1);
1971         }
1972
1973       while (n_left_from > 0 && n_left_to_next > 0)
1974         {
1975           u32 bi0;
1976           vlib_buffer_t * b0;
1977           u32 next0;
1978           u32 sw_if_index0;
1979           ip4_header_t * ip0;
1980           ip_csum_t sum0;
1981           u32 new_addr0, old_addr0;
1982           u16 old_port0, new_port0;
1983           udp_header_t * udp0;
1984           tcp_header_t * tcp0;
1985           icmp46_header_t * icmp0;
1986           snat_session_key_t key0;
1987           u32 rx_fib_index0;
1988           u32 proto0;
1989           snat_session_t * s0 = 0;
1990           clib_bihash_kv_8_8_t kv0, value0;
1991           u32 iph_offset0 = 0;
1992
1993           /* speculatively enqueue b0 to the current next frame */
1994           bi0 = from[0];
1995           to_next[0] = bi0;
1996           from += 1;
1997           to_next += 1;
1998           n_left_from -= 1;
1999           n_left_to_next -= 1;
2000
2001           b0 = vlib_get_buffer (vm, bi0);
2002           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2003
2004           if (is_output_feature)
2005             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
2006
2007           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
2008                  iph_offset0);
2009
2010           udp0 = ip4_next_header (ip0);
2011           tcp0 = (tcp_header_t *) udp0;
2012           icmp0 = (icmp46_header_t *) udp0;
2013
2014           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2015           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
2016                                    sw_if_index0);
2017
2018           if (PREDICT_FALSE(ip0->ttl == 1))
2019             {
2020               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2021               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2022                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2023                                            0);
2024               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2025               goto trace0;
2026             }
2027
2028           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2029
2030           /* Next configured feature, probably ip4-lookup */
2031           if (is_slow_path)
2032             {
2033               if (PREDICT_FALSE (proto0 == ~0))
2034                 {
2035                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
2036                                                   thread_index, now, vm, node);
2037                   if (!s0)
2038                     next0 = SNAT_IN2OUT_NEXT_DROP;
2039                   goto trace0;
2040                 }
2041
2042               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2043                 {
2044                   next0 = icmp_in2out_slow_path
2045                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2046                      next0, now, thread_index, &s0);
2047                   goto trace0;
2048                 }
2049             }
2050           else
2051             {
2052                if (is_output_feature)
2053                 {
2054                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
2055                     goto trace0;
2056                 }
2057
2058               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2059                 {
2060                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2061                   goto trace0;
2062                 }
2063
2064               if (ip4_is_fragment (ip0))
2065                 {
2066                   next0 = SNAT_IN2OUT_NEXT_REASS;
2067                   goto trace0;
2068                 }
2069             }
2070
2071           key0.addr = ip0->src_address;
2072           key0.port = udp0->src_port;
2073           key0.protocol = proto0;
2074           key0.fib_index = rx_fib_index0;
2075
2076           kv0.key = key0.as_u64;
2077
2078           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2079                                       &kv0, &value0))
2080             {
2081               if (is_slow_path)
2082                 {
2083                   if (is_output_feature)
2084                     {
2085                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
2086                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
2087                         goto trace0;
2088                     }
2089                   else
2090                     {
2091                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2092                           ip0, proto0, rx_fib_index0, thread_index)))
2093                         goto trace0;
2094                     }
2095
2096                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2097                                      &s0, node, next0, thread_index);
2098
2099                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2100                     goto trace0;
2101                 }
2102               else
2103                 {
2104                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2105                   goto trace0;
2106                 }
2107             }
2108           else
2109             {
2110               if (PREDICT_FALSE (value0.value == ~0ULL))
2111                 {
2112                   if (is_slow_path)
2113                     {
2114                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2115                                           thread_index, now, vm, node);
2116                       if (!s0 && !sm->forwarding_enabled)
2117                         next0 = SNAT_IN2OUT_NEXT_DROP;
2118                       goto trace0;
2119                     }
2120                   else
2121                     {
2122                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2123                       goto trace0;
2124                     }
2125                 }
2126               else
2127                 {
2128                   s0 = pool_elt_at_index (
2129                     sm->per_thread_data[thread_index].sessions,
2130                     value0.value);
2131                 }
2132             }
2133
2134           b0->flags |= VNET_BUFFER_F_IS_NATED;
2135
2136           old_addr0 = ip0->src_address.as_u32;
2137           ip0->src_address = s0->out2in.addr;
2138           new_addr0 = ip0->src_address.as_u32;
2139           if (!is_output_feature)
2140             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2141
2142           sum0 = ip0->checksum;
2143           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2144                                  ip4_header_t,
2145                                  src_address /* changed member */);
2146           ip0->checksum = ip_csum_fold (sum0);
2147
2148           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2149             {
2150               old_port0 = tcp0->src_port;
2151               tcp0->src_port = s0->out2in.port;
2152               new_port0 = tcp0->src_port;
2153
2154               sum0 = tcp0->checksum;
2155               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2156                                      ip4_header_t,
2157                                      dst_address /* changed member */);
2158               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2159                                      ip4_header_t /* cheat */,
2160                                      length /* changed member */);
2161               tcp0->checksum = ip_csum_fold(sum0);
2162               if (nat44_set_tcp_session_state (sm, s0, tcp0, thread_index))
2163                 goto trace0;
2164             }
2165           else
2166             {
2167               old_port0 = udp0->src_port;
2168               udp0->src_port = s0->out2in.port;
2169               udp0->checksum = 0;
2170             }
2171
2172           /* Accounting */
2173           s0->last_heard = now;
2174           s0->total_pkts++;
2175           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2176           /* Per-user LRU list maintenance */
2177           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2178                              s0->per_user_index);
2179           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2180                               s0->per_user_list_head_index,
2181                               s0->per_user_index);
2182
2183         trace0:
2184           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2185                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2186             {
2187               snat_in2out_trace_t *t =
2188                  vlib_add_trace (vm, node, b0, sizeof (*t));
2189               t->is_slow_path = is_slow_path;
2190               t->sw_if_index = sw_if_index0;
2191               t->next_index = next0;
2192                   t->session_index = ~0;
2193               if (s0)
2194                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2195             }
2196
2197           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2198
2199           /* verify speculative enqueue, maybe switch current next frame */
2200           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2201                                            to_next, n_left_to_next,
2202                                            bi0, next0);
2203         }
2204
2205       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2206     }
2207
2208   vlib_node_increment_counter (vm, stats_node_index,
2209                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2210                                pkts_processed);
2211   return frame->n_vectors;
2212 }
2213
2214 static uword
2215 snat_in2out_fast_path_fn (vlib_main_t * vm,
2216                           vlib_node_runtime_t * node,
2217                           vlib_frame_t * frame)
2218 {
2219   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2220 }
2221
2222 VLIB_REGISTER_NODE (snat_in2out_node) = {
2223   .function = snat_in2out_fast_path_fn,
2224   .name = "nat44-in2out",
2225   .vector_size = sizeof (u32),
2226   .format_trace = format_snat_in2out_trace,
2227   .type = VLIB_NODE_TYPE_INTERNAL,
2228
2229   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2230   .error_strings = snat_in2out_error_strings,
2231
2232   .runtime_data_bytes = sizeof (snat_runtime_t),
2233
2234   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2235
2236   /* edit / add dispositions here */
2237   .next_nodes = {
2238     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2239     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2240     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2241     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2242     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2243   },
2244 };
2245
2246 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2247
2248 static uword
2249 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2250                                  vlib_node_runtime_t * node,
2251                                  vlib_frame_t * frame)
2252 {
2253   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2254 }
2255
2256 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2257   .function = snat_in2out_output_fast_path_fn,
2258   .name = "nat44-in2out-output",
2259   .vector_size = sizeof (u32),
2260   .format_trace = format_snat_in2out_trace,
2261   .type = VLIB_NODE_TYPE_INTERNAL,
2262
2263   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2264   .error_strings = snat_in2out_error_strings,
2265
2266   .runtime_data_bytes = sizeof (snat_runtime_t),
2267
2268   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2269
2270   /* edit / add dispositions here */
2271   .next_nodes = {
2272     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2273     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2274     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2275     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2276     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2277   },
2278 };
2279
2280 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2281                               snat_in2out_output_fast_path_fn);
2282
2283 static uword
2284 snat_in2out_slow_path_fn (vlib_main_t * vm,
2285                           vlib_node_runtime_t * node,
2286                           vlib_frame_t * frame)
2287 {
2288   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2289 }
2290
2291 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2292   .function = snat_in2out_slow_path_fn,
2293   .name = "nat44-in2out-slowpath",
2294   .vector_size = sizeof (u32),
2295   .format_trace = format_snat_in2out_trace,
2296   .type = VLIB_NODE_TYPE_INTERNAL,
2297
2298   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2299   .error_strings = snat_in2out_error_strings,
2300
2301   .runtime_data_bytes = sizeof (snat_runtime_t),
2302
2303   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2304
2305   /* edit / add dispositions here */
2306   .next_nodes = {
2307     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2308     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2309     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2310     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2311     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2312   },
2313 };
2314
2315 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2316                               snat_in2out_slow_path_fn);
2317
2318 static uword
2319 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2320                                  vlib_node_runtime_t * node,
2321                                  vlib_frame_t * frame)
2322 {
2323   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2324 }
2325
2326 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2327   .function = snat_in2out_output_slow_path_fn,
2328   .name = "nat44-in2out-output-slowpath",
2329   .vector_size = sizeof (u32),
2330   .format_trace = format_snat_in2out_trace,
2331   .type = VLIB_NODE_TYPE_INTERNAL,
2332
2333   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2334   .error_strings = snat_in2out_error_strings,
2335
2336   .runtime_data_bytes = sizeof (snat_runtime_t),
2337
2338   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2339
2340   /* edit / add dispositions here */
2341   .next_nodes = {
2342     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2343     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2344     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2345     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2346     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2347   },
2348 };
2349
2350 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2351                               snat_in2out_output_slow_path_fn);
2352
2353 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2354
2355 static uword
2356 nat44_hairpinning_fn (vlib_main_t * vm,
2357                       vlib_node_runtime_t * node,
2358                       vlib_frame_t * frame)
2359 {
2360   u32 n_left_from, * from, * to_next;
2361   snat_in2out_next_t next_index;
2362   u32 pkts_processed = 0;
2363   snat_main_t * sm = &snat_main;
2364   vnet_feature_main_t *fm = &feature_main;
2365   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2366   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2367
2368   from = vlib_frame_vector_args (frame);
2369   n_left_from = frame->n_vectors;
2370   next_index = node->cached_next_index;
2371
2372   while (n_left_from > 0)
2373     {
2374       u32 n_left_to_next;
2375
2376       vlib_get_next_frame (vm, node, next_index,
2377                            to_next, n_left_to_next);
2378
2379       while (n_left_from > 0 && n_left_to_next > 0)
2380         {
2381           u32 bi0;
2382           vlib_buffer_t * b0;
2383           u32 next0;
2384           ip4_header_t * ip0;
2385           u32 proto0;
2386           udp_header_t * udp0;
2387           tcp_header_t * tcp0;
2388
2389           /* speculatively enqueue b0 to the current next frame */
2390           bi0 = from[0];
2391           to_next[0] = bi0;
2392           from += 1;
2393           to_next += 1;
2394           n_left_from -= 1;
2395           n_left_to_next -= 1;
2396
2397           b0 = vlib_get_buffer (vm, bi0);
2398           ip0 = vlib_buffer_get_current (b0);
2399           udp0 = ip4_next_header (ip0);
2400           tcp0 = (tcp_header_t *) udp0;
2401
2402           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2403
2404           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2405                                 &next0, 0);
2406
2407           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2408             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2409
2410           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2411
2412           /* verify speculative enqueue, maybe switch current next frame */
2413           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2414                                            to_next, n_left_to_next,
2415                                            bi0, next0);
2416          }
2417
2418       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2419     }
2420
2421   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2422                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2423                                pkts_processed);
2424   return frame->n_vectors;
2425 }
2426
2427 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2428   .function = nat44_hairpinning_fn,
2429   .name = "nat44-hairpinning",
2430   .vector_size = sizeof (u32),
2431   .type = VLIB_NODE_TYPE_INTERNAL,
2432   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2433   .error_strings = snat_in2out_error_strings,
2434   .n_next_nodes = 2,
2435   .next_nodes = {
2436     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2437     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2438   },
2439 };
2440
2441 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2442                               nat44_hairpinning_fn);
2443
2444 static inline void
2445 nat44_reass_hairpinning (snat_main_t *sm,
2446                          vlib_buffer_t * b0,
2447                          ip4_header_t * ip0,
2448                          u16 sport,
2449                          u16 dport,
2450                          u32 proto0)
2451 {
2452   snat_session_key_t key0, sm0;
2453   snat_session_t * s0;
2454   clib_bihash_kv_8_8_t kv0, value0;
2455   ip_csum_t sum0;
2456   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2457   u16 new_dst_port0, old_dst_port0;
2458   udp_header_t * udp0;
2459   tcp_header_t * tcp0;
2460
2461   key0.addr = ip0->dst_address;
2462   key0.port = dport;
2463   key0.protocol = proto0;
2464   key0.fib_index = sm->outside_fib_index;
2465   kv0.key = key0.as_u64;
2466
2467   udp0 = ip4_next_header (ip0);
2468
2469   /* Check if destination is static mappings */
2470   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2471     {
2472       new_dst_addr0 = sm0.addr.as_u32;
2473       new_dst_port0 = sm0.port;
2474       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2475     }
2476   /* or active sessions */
2477   else
2478     {
2479       if (sm->num_workers > 1)
2480         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2481       else
2482         ti = sm->num_workers;
2483
2484       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2485         {
2486           si = value0.value;
2487           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2488           new_dst_addr0 = s0->in2out.addr.as_u32;
2489           new_dst_port0 = s0->in2out.port;
2490           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2491         }
2492     }
2493
2494   /* Destination is behind the same NAT, use internal address and port */
2495   if (new_dst_addr0)
2496     {
2497       old_dst_addr0 = ip0->dst_address.as_u32;
2498       ip0->dst_address.as_u32 = new_dst_addr0;
2499       sum0 = ip0->checksum;
2500       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2501                              ip4_header_t, dst_address);
2502       ip0->checksum = ip_csum_fold (sum0);
2503
2504       old_dst_port0 = dport;
2505       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2506                        ip4_is_first_fragment (ip0)))
2507         {
2508           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2509             {
2510               tcp0 = ip4_next_header (ip0);
2511               tcp0->dst = new_dst_port0;
2512               sum0 = tcp0->checksum;
2513               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2514                                      ip4_header_t, dst_address);
2515               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2516                                      ip4_header_t /* cheat */, length);
2517               tcp0->checksum = ip_csum_fold(sum0);
2518             }
2519           else
2520             {
2521               udp0->dst_port = new_dst_port0;
2522               udp0->checksum = 0;
2523             }
2524         }
2525       else
2526         {
2527           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2528             {
2529               tcp0 = ip4_next_header (ip0);
2530               sum0 = tcp0->checksum;
2531               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2532                                      ip4_header_t, dst_address);
2533               tcp0->checksum = ip_csum_fold(sum0);
2534             }
2535         }
2536     }
2537 }
2538
2539 static uword
2540 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2541                             vlib_node_runtime_t * node,
2542                             vlib_frame_t * frame)
2543 {
2544   u32 n_left_from, *from, *to_next;
2545   snat_in2out_next_t next_index;
2546   u32 pkts_processed = 0;
2547   snat_main_t *sm = &snat_main;
2548   f64 now = vlib_time_now (vm);
2549   u32 thread_index = vlib_get_thread_index ();
2550   snat_main_per_thread_data_t *per_thread_data =
2551     &sm->per_thread_data[thread_index];
2552   u32 *fragments_to_drop = 0;
2553   u32 *fragments_to_loopback = 0;
2554
2555   from = vlib_frame_vector_args (frame);
2556   n_left_from = frame->n_vectors;
2557   next_index = node->cached_next_index;
2558
2559   while (n_left_from > 0)
2560     {
2561       u32 n_left_to_next;
2562
2563       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2564
2565       while (n_left_from > 0 && n_left_to_next > 0)
2566        {
2567           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2568           vlib_buffer_t *b0;
2569           u32 next0;
2570           u8 cached0 = 0;
2571           ip4_header_t *ip0;
2572           nat_reass_ip4_t *reass0;
2573           udp_header_t * udp0;
2574           tcp_header_t * tcp0;
2575           snat_session_key_t key0;
2576           clib_bihash_kv_8_8_t kv0, value0;
2577           snat_session_t * s0 = 0;
2578           u16 old_port0, new_port0;
2579           ip_csum_t sum0;
2580
2581           /* speculatively enqueue b0 to the current next frame */
2582           bi0 = from[0];
2583           to_next[0] = bi0;
2584           from += 1;
2585           to_next += 1;
2586           n_left_from -= 1;
2587           n_left_to_next -= 1;
2588
2589           b0 = vlib_get_buffer (vm, bi0);
2590           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2591
2592           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2593           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2594                                                                sw_if_index0);
2595
2596           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2597             {
2598               next0 = SNAT_IN2OUT_NEXT_DROP;
2599               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2600               goto trace0;
2601             }
2602
2603           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2604           udp0 = ip4_next_header (ip0);
2605           tcp0 = (tcp_header_t *) udp0;
2606           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2607
2608           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2609                                                  ip0->dst_address,
2610                                                  ip0->fragment_id,
2611                                                  ip0->protocol,
2612                                                  1,
2613                                                  &fragments_to_drop);
2614
2615           if (PREDICT_FALSE (!reass0))
2616             {
2617               next0 = SNAT_IN2OUT_NEXT_DROP;
2618               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2619               goto trace0;
2620             }
2621
2622           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2623             {
2624               key0.addr = ip0->src_address;
2625               key0.port = udp0->src_port;
2626               key0.protocol = proto0;
2627               key0.fib_index = rx_fib_index0;
2628               kv0.key = key0.as_u64;
2629
2630               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2631                 {
2632                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2633                       ip0, proto0, rx_fib_index0, thread_index)))
2634                     goto trace0;
2635
2636                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2637                                      &s0, node, next0, thread_index);
2638
2639                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2640                     goto trace0;
2641
2642                   reass0->sess_index = s0 - per_thread_data->sessions;
2643                 }
2644               else
2645                 {
2646                   s0 = pool_elt_at_index (per_thread_data->sessions,
2647                                           value0.value);
2648                   reass0->sess_index = value0.value;
2649                 }
2650               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2651             }
2652           else
2653             {
2654               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2655                 {
2656                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2657                     {
2658                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2659                       next0 = SNAT_IN2OUT_NEXT_DROP;
2660                       goto trace0;
2661                     }
2662                   cached0 = 1;
2663                   goto trace0;
2664                 }
2665               s0 = pool_elt_at_index (per_thread_data->sessions,
2666                                       reass0->sess_index);
2667             }
2668
2669           old_addr0 = ip0->src_address.as_u32;
2670           ip0->src_address = s0->out2in.addr;
2671           new_addr0 = ip0->src_address.as_u32;
2672           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2673
2674           sum0 = ip0->checksum;
2675           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2676                                  ip4_header_t,
2677                                  src_address /* changed member */);
2678           ip0->checksum = ip_csum_fold (sum0);
2679
2680           /* Hairpinning */
2681           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2682                                    s0->ext_host_port, proto0);
2683
2684           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2685             {
2686               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2687                 {
2688                   old_port0 = tcp0->src_port;
2689                   tcp0->src_port = s0->out2in.port;
2690                   new_port0 = tcp0->src_port;
2691
2692                   sum0 = tcp0->checksum;
2693                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2694                                          ip4_header_t,
2695                                          dst_address /* changed member */);
2696                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2697                                          ip4_header_t /* cheat */,
2698                                          length /* changed member */);
2699                   tcp0->checksum = ip_csum_fold(sum0);
2700                   if (nat44_set_tcp_session_state (sm, s0, tcp0, thread_index))
2701                     goto trace0;
2702                 }
2703               else
2704                 {
2705                   old_port0 = udp0->src_port;
2706                   udp0->src_port = s0->out2in.port;
2707                   udp0->checksum = 0;
2708                 }
2709             }
2710
2711           /* Accounting */
2712           s0->last_heard = now;
2713           s0->total_pkts++;
2714           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2715           /* Per-user LRU list maintenance */
2716           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2717                              s0->per_user_index);
2718           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2719                               s0->per_user_list_head_index,
2720                               s0->per_user_index);
2721
2722         trace0:
2723           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2724                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2725             {
2726               nat44_in2out_reass_trace_t *t =
2727                  vlib_add_trace (vm, node, b0, sizeof (*t));
2728               t->cached = cached0;
2729               t->sw_if_index = sw_if_index0;
2730               t->next_index = next0;
2731             }
2732
2733           if (cached0)
2734             {
2735               n_left_to_next++;
2736               to_next--;
2737             }
2738           else
2739             {
2740               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2741
2742               /* verify speculative enqueue, maybe switch current next frame */
2743               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2744                                                to_next, n_left_to_next,
2745                                                bi0, next0);
2746             }
2747
2748           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2749             {
2750               from = vlib_frame_vector_args (frame);
2751               u32 len = vec_len (fragments_to_loopback);
2752               if (len <= VLIB_FRAME_SIZE)
2753                 {
2754                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2755                   n_left_from = len;
2756                   vec_reset_length (fragments_to_loopback);
2757                 }
2758               else
2759                 {
2760                   clib_memcpy (from,
2761                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2762                                sizeof (u32) * VLIB_FRAME_SIZE);
2763                   n_left_from = VLIB_FRAME_SIZE;
2764                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2765                 }
2766             }
2767        }
2768
2769       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2770     }
2771
2772   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2773                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2774                                pkts_processed);
2775
2776   nat_send_all_to_node (vm, fragments_to_drop, node,
2777                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2778                         SNAT_IN2OUT_NEXT_DROP);
2779
2780   vec_free (fragments_to_drop);
2781   vec_free (fragments_to_loopback);
2782   return frame->n_vectors;
2783 }
2784
2785 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2786   .function = nat44_in2out_reass_node_fn,
2787   .name = "nat44-in2out-reass",
2788   .vector_size = sizeof (u32),
2789   .format_trace = format_nat44_in2out_reass_trace,
2790   .type = VLIB_NODE_TYPE_INTERNAL,
2791
2792   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2793   .error_strings = snat_in2out_error_strings,
2794
2795   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2796   .next_nodes = {
2797     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2798     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2799     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2800     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2801     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2802   },
2803 };
2804
2805 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2806                               nat44_in2out_reass_node_fn);
2807
2808 /**************************/
2809 /*** deterministic mode ***/
2810 /**************************/
2811 static uword
2812 snat_det_in2out_node_fn (vlib_main_t * vm,
2813                          vlib_node_runtime_t * node,
2814                          vlib_frame_t * frame)
2815 {
2816   u32 n_left_from, * from, * to_next;
2817   snat_in2out_next_t next_index;
2818   u32 pkts_processed = 0;
2819   snat_main_t * sm = &snat_main;
2820   u32 now = (u32) vlib_time_now (vm);
2821   u32 thread_index = vlib_get_thread_index ();
2822
2823   from = vlib_frame_vector_args (frame);
2824   n_left_from = frame->n_vectors;
2825   next_index = node->cached_next_index;
2826
2827   while (n_left_from > 0)
2828     {
2829       u32 n_left_to_next;
2830
2831       vlib_get_next_frame (vm, node, next_index,
2832                            to_next, n_left_to_next);
2833
2834       while (n_left_from >= 4 && n_left_to_next >= 2)
2835         {
2836           u32 bi0, bi1;
2837           vlib_buffer_t * b0, * b1;
2838           u32 next0, next1;
2839           u32 sw_if_index0, sw_if_index1;
2840           ip4_header_t * ip0, * ip1;
2841           ip_csum_t sum0, sum1;
2842           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2843           u16 old_port0, new_port0, lo_port0, i0;
2844           u16 old_port1, new_port1, lo_port1, i1;
2845           udp_header_t * udp0, * udp1;
2846           tcp_header_t * tcp0, * tcp1;
2847           u32 proto0, proto1;
2848           snat_det_out_key_t key0, key1;
2849           snat_det_map_t * dm0, * dm1;
2850           snat_det_session_t * ses0 = 0, * ses1 = 0;
2851           u32 rx_fib_index0, rx_fib_index1;
2852           icmp46_header_t * icmp0, * icmp1;
2853
2854           /* Prefetch next iteration. */
2855           {
2856             vlib_buffer_t * p2, * p3;
2857
2858             p2 = vlib_get_buffer (vm, from[2]);
2859             p3 = vlib_get_buffer (vm, from[3]);
2860
2861             vlib_prefetch_buffer_header (p2, LOAD);
2862             vlib_prefetch_buffer_header (p3, LOAD);
2863
2864             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2865             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2866           }
2867
2868           /* speculatively enqueue b0 and b1 to the current next frame */
2869           to_next[0] = bi0 = from[0];
2870           to_next[1] = bi1 = from[1];
2871           from += 2;
2872           to_next += 2;
2873           n_left_from -= 2;
2874           n_left_to_next -= 2;
2875
2876           b0 = vlib_get_buffer (vm, bi0);
2877           b1 = vlib_get_buffer (vm, bi1);
2878
2879           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2880           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2881
2882           ip0 = vlib_buffer_get_current (b0);
2883           udp0 = ip4_next_header (ip0);
2884           tcp0 = (tcp_header_t *) udp0;
2885
2886           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2887
2888           if (PREDICT_FALSE(ip0->ttl == 1))
2889             {
2890               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2891               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2892                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2893                                            0);
2894               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2895               goto trace0;
2896             }
2897
2898           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2899
2900           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2901             {
2902               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2903               icmp0 = (icmp46_header_t *) udp0;
2904
2905               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2906                                   rx_fib_index0, node, next0, thread_index,
2907                                   &ses0, &dm0);
2908               goto trace0;
2909             }
2910
2911           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2912           if (PREDICT_FALSE(!dm0))
2913             {
2914               clib_warning("no match for internal host %U",
2915                            format_ip4_address, &ip0->src_address);
2916               next0 = SNAT_IN2OUT_NEXT_DROP;
2917               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2918               goto trace0;
2919             }
2920
2921           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2922
2923           key0.ext_host_addr = ip0->dst_address;
2924           key0.ext_host_port = tcp0->dst;
2925
2926           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2927           if (PREDICT_FALSE(!ses0))
2928             {
2929               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2930                 {
2931                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2932                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2933
2934                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2935                     continue;
2936
2937                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2938                   break;
2939                 }
2940               if (PREDICT_FALSE(!ses0))
2941                 {
2942                   /* too many sessions for user, send ICMP error packet */
2943
2944                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2945                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2946                                                ICMP4_destination_unreachable_destination_unreachable_host,
2947                                                0);
2948                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2949                   goto trace0;
2950                 }
2951             }
2952
2953           new_port0 = ses0->out.out_port;
2954
2955           old_addr0.as_u32 = ip0->src_address.as_u32;
2956           ip0->src_address.as_u32 = new_addr0.as_u32;
2957           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2958
2959           sum0 = ip0->checksum;
2960           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2961                                  ip4_header_t,
2962                                  src_address /* changed member */);
2963           ip0->checksum = ip_csum_fold (sum0);
2964
2965           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2966             {
2967               if (tcp0->flags & TCP_FLAG_SYN)
2968                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2969               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2970                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2971               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2972                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2973               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2974                 snat_det_ses_close(dm0, ses0);
2975               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2976                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2977               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2978                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2979
2980               old_port0 = tcp0->src;
2981               tcp0->src = new_port0;
2982
2983               sum0 = tcp0->checksum;
2984               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2985                                      ip4_header_t,
2986                                      dst_address /* changed member */);
2987               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2988                                      ip4_header_t /* cheat */,
2989                                      length /* changed member */);
2990               tcp0->checksum = ip_csum_fold(sum0);
2991             }
2992           else
2993             {
2994               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2995               old_port0 = udp0->src_port;
2996               udp0->src_port = new_port0;
2997               udp0->checksum = 0;
2998             }
2999
3000           switch(ses0->state)
3001             {
3002             case SNAT_SESSION_UDP_ACTIVE:
3003                 ses0->expire = now + sm->udp_timeout;
3004                 break;
3005             case SNAT_SESSION_TCP_SYN_SENT:
3006             case SNAT_SESSION_TCP_FIN_WAIT:
3007             case SNAT_SESSION_TCP_CLOSE_WAIT:
3008             case SNAT_SESSION_TCP_LAST_ACK:
3009                 ses0->expire = now + sm->tcp_transitory_timeout;
3010                 break;
3011             case SNAT_SESSION_TCP_ESTABLISHED:
3012                 ses0->expire = now + sm->tcp_established_timeout;
3013                 break;
3014             }
3015
3016         trace0:
3017           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3018                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3019             {
3020               snat_in2out_trace_t *t =
3021                  vlib_add_trace (vm, node, b0, sizeof (*t));
3022               t->is_slow_path = 0;
3023               t->sw_if_index = sw_if_index0;
3024               t->next_index = next0;
3025               t->session_index = ~0;
3026               if (ses0)
3027                 t->session_index = ses0 - dm0->sessions;
3028             }
3029
3030           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3031
3032           ip1 = vlib_buffer_get_current (b1);
3033           udp1 = ip4_next_header (ip1);
3034           tcp1 = (tcp_header_t *) udp1;
3035
3036           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3037
3038           if (PREDICT_FALSE(ip1->ttl == 1))
3039             {
3040               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3041               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3042                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3043                                            0);
3044               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3045               goto trace1;
3046             }
3047
3048           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3049
3050           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3051             {
3052               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3053               icmp1 = (icmp46_header_t *) udp1;
3054
3055               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
3056                                   rx_fib_index1, node, next1, thread_index,
3057                                   &ses1, &dm1);
3058               goto trace1;
3059             }
3060
3061           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
3062           if (PREDICT_FALSE(!dm1))
3063             {
3064               clib_warning("no match for internal host %U",
3065                            format_ip4_address, &ip0->src_address);
3066               next1 = SNAT_IN2OUT_NEXT_DROP;
3067               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3068               goto trace1;
3069             }
3070
3071           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3072
3073           key1.ext_host_addr = ip1->dst_address;
3074           key1.ext_host_port = tcp1->dst;
3075
3076           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3077           if (PREDICT_FALSE(!ses1))
3078             {
3079               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3080                 {
3081                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
3082                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3083
3084                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3085                     continue;
3086
3087                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3088                   break;
3089                 }
3090               if (PREDICT_FALSE(!ses1))
3091                 {
3092                   /* too many sessions for user, send ICMP error packet */
3093
3094                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3095                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3096                                                ICMP4_destination_unreachable_destination_unreachable_host,
3097                                                0);
3098                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3099                   goto trace1;
3100                 }
3101             }
3102
3103           new_port1 = ses1->out.out_port;
3104
3105           old_addr1.as_u32 = ip1->src_address.as_u32;
3106           ip1->src_address.as_u32 = new_addr1.as_u32;
3107           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3108
3109           sum1 = ip1->checksum;
3110           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3111                                  ip4_header_t,
3112                                  src_address /* changed member */);
3113           ip1->checksum = ip_csum_fold (sum1);
3114
3115           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3116             {
3117               if (tcp1->flags & TCP_FLAG_SYN)
3118                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3119               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3120                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3121               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3122                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3123               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3124                 snat_det_ses_close(dm1, ses1);
3125               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3126                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3127               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3128                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3129
3130               old_port1 = tcp1->src;
3131               tcp1->src = new_port1;
3132
3133               sum1 = tcp1->checksum;
3134               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3135                                      ip4_header_t,
3136                                      dst_address /* changed member */);
3137               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3138                                      ip4_header_t /* cheat */,
3139                                      length /* changed member */);
3140               tcp1->checksum = ip_csum_fold(sum1);
3141             }
3142           else
3143             {
3144               ses1->state = SNAT_SESSION_UDP_ACTIVE;
3145               old_port1 = udp1->src_port;
3146               udp1->src_port = new_port1;
3147               udp1->checksum = 0;
3148             }
3149
3150           switch(ses1->state)
3151             {
3152             case SNAT_SESSION_UDP_ACTIVE:
3153                 ses1->expire = now + sm->udp_timeout;
3154                 break;
3155             case SNAT_SESSION_TCP_SYN_SENT:
3156             case SNAT_SESSION_TCP_FIN_WAIT:
3157             case SNAT_SESSION_TCP_CLOSE_WAIT:
3158             case SNAT_SESSION_TCP_LAST_ACK:
3159                 ses1->expire = now + sm->tcp_transitory_timeout;
3160                 break;
3161             case SNAT_SESSION_TCP_ESTABLISHED:
3162                 ses1->expire = now + sm->tcp_established_timeout;
3163                 break;
3164             }
3165
3166         trace1:
3167           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3168                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3169             {
3170               snat_in2out_trace_t *t =
3171                  vlib_add_trace (vm, node, b1, sizeof (*t));
3172               t->is_slow_path = 0;
3173               t->sw_if_index = sw_if_index1;
3174               t->next_index = next1;
3175               t->session_index = ~0;
3176               if (ses1)
3177                 t->session_index = ses1 - dm1->sessions;
3178             }
3179
3180           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3181
3182           /* verify speculative enqueues, maybe switch current next frame */
3183           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3184                                            to_next, n_left_to_next,
3185                                            bi0, bi1, next0, next1);
3186          }
3187
3188       while (n_left_from > 0 && n_left_to_next > 0)
3189         {
3190           u32 bi0;
3191           vlib_buffer_t * b0;
3192           u32 next0;
3193           u32 sw_if_index0;
3194           ip4_header_t * ip0;
3195           ip_csum_t sum0;
3196           ip4_address_t new_addr0, old_addr0;
3197           u16 old_port0, new_port0, lo_port0, i0;
3198           udp_header_t * udp0;
3199           tcp_header_t * tcp0;
3200           u32 proto0;
3201           snat_det_out_key_t key0;
3202           snat_det_map_t * dm0;
3203           snat_det_session_t * ses0 = 0;
3204           u32 rx_fib_index0;
3205           icmp46_header_t * icmp0;
3206
3207           /* speculatively enqueue b0 to the current next frame */
3208           bi0 = from[0];
3209           to_next[0] = bi0;
3210           from += 1;
3211           to_next += 1;
3212           n_left_from -= 1;
3213           n_left_to_next -= 1;
3214
3215           b0 = vlib_get_buffer (vm, bi0);
3216           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3217
3218           ip0 = vlib_buffer_get_current (b0);
3219           udp0 = ip4_next_header (ip0);
3220           tcp0 = (tcp_header_t *) udp0;
3221
3222           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3223
3224           if (PREDICT_FALSE(ip0->ttl == 1))
3225             {
3226               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3227               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3228                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3229                                            0);
3230               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3231               goto trace00;
3232             }
3233
3234           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3235
3236           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3237             {
3238               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3239               icmp0 = (icmp46_header_t *) udp0;
3240
3241               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3242                                   rx_fib_index0, node, next0, thread_index,
3243                                   &ses0, &dm0);
3244               goto trace00;
3245             }
3246
3247           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3248           if (PREDICT_FALSE(!dm0))
3249             {
3250               clib_warning("no match for internal host %U",
3251                            format_ip4_address, &ip0->src_address);
3252               next0 = SNAT_IN2OUT_NEXT_DROP;
3253               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3254               goto trace00;
3255             }
3256
3257           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3258
3259           key0.ext_host_addr = ip0->dst_address;
3260           key0.ext_host_port = tcp0->dst;
3261
3262           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3263           if (PREDICT_FALSE(!ses0))
3264             {
3265               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3266                 {
3267                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3268                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3269
3270                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3271                     continue;
3272
3273                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3274                   break;
3275                 }
3276               if (PREDICT_FALSE(!ses0))
3277                 {
3278                   /* too many sessions for user, send ICMP error packet */
3279
3280                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3281                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3282                                                ICMP4_destination_unreachable_destination_unreachable_host,
3283                                                0);
3284                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3285                   goto trace00;
3286                 }
3287             }
3288
3289           new_port0 = ses0->out.out_port;
3290
3291           old_addr0.as_u32 = ip0->src_address.as_u32;
3292           ip0->src_address.as_u32 = new_addr0.as_u32;
3293           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3294
3295           sum0 = ip0->checksum;
3296           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3297                                  ip4_header_t,
3298                                  src_address /* changed member */);
3299           ip0->checksum = ip_csum_fold (sum0);
3300
3301           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3302             {
3303               if (tcp0->flags & TCP_FLAG_SYN)
3304                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3305               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3306                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3307               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3308                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3309               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3310                 snat_det_ses_close(dm0, ses0);
3311               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3312                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3313               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3314                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3315
3316               old_port0 = tcp0->src;
3317               tcp0->src = new_port0;
3318
3319               sum0 = tcp0->checksum;
3320               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3321                                      ip4_header_t,
3322                                      dst_address /* changed member */);
3323               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3324                                      ip4_header_t /* cheat */,
3325                                      length /* changed member */);
3326               tcp0->checksum = ip_csum_fold(sum0);
3327             }
3328           else
3329             {
3330               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3331               old_port0 = udp0->src_port;
3332               udp0->src_port = new_port0;
3333               udp0->checksum = 0;
3334             }
3335
3336           switch(ses0->state)
3337             {
3338             case SNAT_SESSION_UDP_ACTIVE:
3339                 ses0->expire = now + sm->udp_timeout;
3340                 break;
3341             case SNAT_SESSION_TCP_SYN_SENT:
3342             case SNAT_SESSION_TCP_FIN_WAIT:
3343             case SNAT_SESSION_TCP_CLOSE_WAIT:
3344             case SNAT_SESSION_TCP_LAST_ACK:
3345                 ses0->expire = now + sm->tcp_transitory_timeout;
3346                 break;
3347             case SNAT_SESSION_TCP_ESTABLISHED:
3348                 ses0->expire = now + sm->tcp_established_timeout;
3349                 break;
3350             }
3351
3352         trace00:
3353           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3354                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3355             {
3356               snat_in2out_trace_t *t =
3357                  vlib_add_trace (vm, node, b0, sizeof (*t));
3358               t->is_slow_path = 0;
3359               t->sw_if_index = sw_if_index0;
3360               t->next_index = next0;
3361               t->session_index = ~0;
3362               if (ses0)
3363                 t->session_index = ses0 - dm0->sessions;
3364             }
3365
3366           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3367
3368           /* verify speculative enqueue, maybe switch current next frame */
3369           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3370                                            to_next, n_left_to_next,
3371                                            bi0, next0);
3372         }
3373
3374       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3375     }
3376
3377   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3378                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3379                                pkts_processed);
3380   return frame->n_vectors;
3381 }
3382
3383 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3384   .function = snat_det_in2out_node_fn,
3385   .name = "nat44-det-in2out",
3386   .vector_size = sizeof (u32),
3387   .format_trace = format_snat_in2out_trace,
3388   .type = VLIB_NODE_TYPE_INTERNAL,
3389
3390   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3391   .error_strings = snat_in2out_error_strings,
3392
3393   .runtime_data_bytes = sizeof (snat_runtime_t),
3394
3395   .n_next_nodes = 3,
3396
3397   /* edit / add dispositions here */
3398   .next_nodes = {
3399     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3400     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3401     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3402   },
3403 };
3404
3405 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3406
3407 /**
3408  * Get address and port values to be used for ICMP packet translation
3409  * and create session if needed
3410  *
3411  * @param[in,out] sm             NAT main
3412  * @param[in,out] node           NAT node runtime
3413  * @param[in] thread_index       thread index
3414  * @param[in,out] b0             buffer containing packet to be translated
3415  * @param[out] p_proto           protocol used for matching
3416  * @param[out] p_value           address and port after NAT translation
3417  * @param[out] p_dont_translate  if packet should not be translated
3418  * @param d                      optional parameter
3419  * @param e                      optional parameter
3420  */
3421 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3422                           u32 thread_index, vlib_buffer_t *b0,
3423                           ip4_header_t *ip0, u8 *p_proto,
3424                           snat_session_key_t *p_value,
3425                           u8 *p_dont_translate, void *d, void *e)
3426 {
3427   icmp46_header_t *icmp0;
3428   u32 sw_if_index0;
3429   u32 rx_fib_index0;
3430   u8 protocol;
3431   snat_det_out_key_t key0;
3432   u8 dont_translate = 0;
3433   u32 next0 = ~0;
3434   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3435   ip4_header_t *inner_ip0;
3436   void *l4_header = 0;
3437   icmp46_header_t *inner_icmp0;
3438   snat_det_map_t * dm0 = 0;
3439   ip4_address_t new_addr0;
3440   u16 lo_port0, i0;
3441   snat_det_session_t * ses0 = 0;
3442   ip4_address_t in_addr;
3443   u16 in_port;
3444
3445   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3446   echo0 = (icmp_echo_header_t *)(icmp0+1);
3447   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3448   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3449
3450   if (!icmp_is_error_message (icmp0))
3451     {
3452       protocol = SNAT_PROTOCOL_ICMP;
3453       in_addr = ip0->src_address;
3454       in_port = echo0->identifier;
3455     }
3456   else
3457     {
3458       inner_ip0 = (ip4_header_t *)(echo0+1);
3459       l4_header = ip4_next_header (inner_ip0);
3460       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3461       in_addr = inner_ip0->dst_address;
3462       switch (protocol)
3463         {
3464         case SNAT_PROTOCOL_ICMP:
3465           inner_icmp0 = (icmp46_header_t*)l4_header;
3466           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3467           in_port = inner_echo0->identifier;
3468           break;
3469         case SNAT_PROTOCOL_UDP:
3470         case SNAT_PROTOCOL_TCP:
3471           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3472           break;
3473         default:
3474           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3475           next0 = SNAT_IN2OUT_NEXT_DROP;
3476           goto out;
3477         }
3478     }
3479
3480   dm0 = snat_det_map_by_user(sm, &in_addr);
3481   if (PREDICT_FALSE(!dm0))
3482     {
3483       clib_warning("no match for internal host %U",
3484                    format_ip4_address, &in_addr);
3485       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3486           IP_PROTOCOL_ICMP, rx_fib_index0)))
3487         {
3488           dont_translate = 1;
3489           goto out;
3490         }
3491       next0 = SNAT_IN2OUT_NEXT_DROP;
3492       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3493       goto out;
3494     }
3495
3496   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3497
3498   key0.ext_host_addr = ip0->dst_address;
3499   key0.ext_host_port = 0;
3500
3501   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3502   if (PREDICT_FALSE(!ses0))
3503     {
3504       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3505           IP_PROTOCOL_ICMP, rx_fib_index0)))
3506         {
3507           dont_translate = 1;
3508           goto out;
3509         }
3510       if (icmp0->type != ICMP4_echo_request)
3511         {
3512           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3513           next0 = SNAT_IN2OUT_NEXT_DROP;
3514           goto out;
3515         }
3516       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3517         {
3518           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3519             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3520
3521           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3522             continue;
3523
3524           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3525           break;
3526         }
3527       if (PREDICT_FALSE(!ses0))
3528         {
3529           next0 = SNAT_IN2OUT_NEXT_DROP;
3530           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3531           goto out;
3532         }
3533     }
3534
3535   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3536                     !icmp_is_error_message (icmp0)))
3537     {
3538       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3539       next0 = SNAT_IN2OUT_NEXT_DROP;
3540       goto out;
3541     }
3542
3543   u32 now = (u32) vlib_time_now (sm->vlib_main);
3544
3545   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3546   ses0->expire = now + sm->icmp_timeout;
3547
3548 out:
3549   *p_proto = protocol;
3550   if (ses0)
3551     {
3552       p_value->addr = new_addr0;
3553       p_value->fib_index = sm->outside_fib_index;
3554       p_value->port = ses0->out.out_port;
3555     }
3556   *p_dont_translate = dont_translate;
3557   if (d)
3558     *(snat_det_session_t**)d = ses0;
3559   if (e)
3560     *(snat_det_map_t**)e = dm0;
3561   return next0;
3562 }
3563
3564 /**********************/
3565 /*** worker handoff ***/
3566 /**********************/
3567 static inline uword
3568 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3569                                       vlib_node_runtime_t * node,
3570                                       vlib_frame_t * frame,
3571                                       u8 is_output)
3572 {
3573   snat_main_t *sm = &snat_main;
3574   vlib_thread_main_t *tm = vlib_get_thread_main ();
3575   u32 n_left_from, *from, *to_next = 0;
3576   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3577   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3578     = 0;
3579   vlib_frame_queue_elt_t *hf = 0;
3580   vlib_frame_t *f = 0;
3581   int i;
3582   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3583   u32 next_worker_index = 0;
3584   u32 current_worker_index = ~0;
3585   u32 thread_index = vlib_get_thread_index ();
3586   u32 fq_index;
3587   u32 to_node_index;
3588
3589   ASSERT (vec_len (sm->workers));
3590
3591   if (is_output)
3592     {
3593       fq_index = sm->fq_in2out_output_index;
3594       to_node_index = sm->in2out_output_node_index;
3595     }
3596   else
3597     {
3598       fq_index = sm->fq_in2out_index;
3599       to_node_index = sm->in2out_node_index;
3600     }
3601
3602   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3603     {
3604       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3605
3606       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3607                                sm->first_worker_index + sm->num_workers - 1,
3608                                (vlib_frame_queue_t *) (~0));
3609     }
3610
3611   from = vlib_frame_vector_args (frame);
3612   n_left_from = frame->n_vectors;
3613
3614   while (n_left_from > 0)
3615     {
3616       u32 bi0;
3617       vlib_buffer_t *b0;
3618       u32 sw_if_index0;
3619       u32 rx_fib_index0;
3620       ip4_header_t * ip0;
3621       u8 do_handoff;
3622
3623       bi0 = from[0];
3624       from += 1;
3625       n_left_from -= 1;
3626
3627       b0 = vlib_get_buffer (vm, bi0);
3628
3629       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3630       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3631
3632       ip0 = vlib_buffer_get_current (b0);
3633
3634       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3635
3636       if (PREDICT_FALSE (next_worker_index != thread_index))
3637         {
3638           do_handoff = 1;
3639
3640           if (next_worker_index != current_worker_index)
3641             {
3642               if (hf)
3643                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3644
3645               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3646                                                       next_worker_index,
3647                                                       handoff_queue_elt_by_worker_index);
3648
3649               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3650               to_next_worker = &hf->buffer_index[hf->n_vectors];
3651               current_worker_index = next_worker_index;
3652             }
3653
3654           /* enqueue to correct worker thread */
3655           to_next_worker[0] = bi0;
3656           to_next_worker++;
3657           n_left_to_next_worker--;
3658
3659           if (n_left_to_next_worker == 0)
3660             {
3661               hf->n_vectors = VLIB_FRAME_SIZE;
3662               vlib_put_frame_queue_elt (hf);
3663               current_worker_index = ~0;
3664               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3665               hf = 0;
3666             }
3667         }
3668       else
3669         {
3670           do_handoff = 0;
3671           /* if this is 1st frame */
3672           if (!f)
3673             {
3674               f = vlib_get_frame_to_node (vm, to_node_index);
3675               to_next = vlib_frame_vector_args (f);
3676             }
3677
3678           to_next[0] = bi0;
3679           to_next += 1;
3680           f->n_vectors++;
3681         }
3682
3683       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3684                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3685         {
3686           snat_in2out_worker_handoff_trace_t *t =
3687             vlib_add_trace (vm, node, b0, sizeof (*t));
3688           t->next_worker_index = next_worker_index;
3689           t->do_handoff = do_handoff;
3690         }
3691     }
3692
3693   if (f)
3694     vlib_put_frame_to_node (vm, to_node_index, f);
3695
3696   if (hf)
3697     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3698
3699   /* Ship frames to the worker nodes */
3700   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3701     {
3702       if (handoff_queue_elt_by_worker_index[i])
3703         {
3704           hf = handoff_queue_elt_by_worker_index[i];
3705           /*
3706            * It works better to let the handoff node
3707            * rate-adapt, always ship the handoff queue element.
3708            */
3709           if (1 || hf->n_vectors == hf->last_n_vectors)
3710             {
3711               vlib_put_frame_queue_elt (hf);
3712               handoff_queue_elt_by_worker_index[i] = 0;
3713             }
3714           else
3715             hf->last_n_vectors = hf->n_vectors;
3716         }
3717       congested_handoff_queue_by_worker_index[i] =
3718         (vlib_frame_queue_t *) (~0);
3719     }
3720   hf = 0;
3721   current_worker_index = ~0;
3722   return frame->n_vectors;
3723 }
3724
3725 static uword
3726 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3727                                vlib_node_runtime_t * node,
3728                                vlib_frame_t * frame)
3729 {
3730   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3731 }
3732
3733 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3734   .function = snat_in2out_worker_handoff_fn,
3735   .name = "nat44-in2out-worker-handoff",
3736   .vector_size = sizeof (u32),
3737   .format_trace = format_snat_in2out_worker_handoff_trace,
3738   .type = VLIB_NODE_TYPE_INTERNAL,
3739
3740   .n_next_nodes = 1,
3741
3742   .next_nodes = {
3743     [0] = "error-drop",
3744   },
3745 };
3746
3747 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3748                               snat_in2out_worker_handoff_fn);
3749
3750 static uword
3751 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3752                                       vlib_node_runtime_t * node,
3753                                       vlib_frame_t * frame)
3754 {
3755   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3756 }
3757
3758 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3759   .function = snat_in2out_output_worker_handoff_fn,
3760   .name = "nat44-in2out-output-worker-handoff",
3761   .vector_size = sizeof (u32),
3762   .format_trace = format_snat_in2out_worker_handoff_trace,
3763   .type = VLIB_NODE_TYPE_INTERNAL,
3764
3765   .n_next_nodes = 1,
3766
3767   .next_nodes = {
3768     [0] = "error-drop",
3769   },
3770 };
3771
3772 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3773                               snat_in2out_output_worker_handoff_fn);
3774
3775 static_always_inline int
3776 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3777 {
3778   snat_address_t * ap;
3779   clib_bihash_kv_8_8_t kv, value;
3780   snat_session_key_t m_key;
3781
3782   vec_foreach (ap, sm->addresses)
3783     {
3784       if (ap->addr.as_u32 == dst_addr->as_u32)
3785         return 1;
3786     }
3787
3788   m_key.addr.as_u32 = dst_addr->as_u32;
3789   m_key.fib_index = sm->outside_fib_index;
3790   m_key.port = 0;
3791   m_key.protocol = 0;
3792   kv.key = m_key.as_u64;
3793   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3794     return 1;
3795
3796   return 0;
3797 }
3798
3799 static uword
3800 snat_hairpin_dst_fn (vlib_main_t * vm,
3801                      vlib_node_runtime_t * node,
3802                      vlib_frame_t * frame)
3803 {
3804   u32 n_left_from, * from, * to_next;
3805   snat_in2out_next_t next_index;
3806   u32 pkts_processed = 0;
3807   snat_main_t * sm = &snat_main;
3808
3809   from = vlib_frame_vector_args (frame);
3810   n_left_from = frame->n_vectors;
3811   next_index = node->cached_next_index;
3812
3813   while (n_left_from > 0)
3814     {
3815       u32 n_left_to_next;
3816
3817       vlib_get_next_frame (vm, node, next_index,
3818                            to_next, n_left_to_next);
3819
3820       while (n_left_from > 0 && n_left_to_next > 0)
3821         {
3822           u32 bi0;
3823           vlib_buffer_t * b0;
3824           u32 next0;
3825           ip4_header_t * ip0;
3826           u32 proto0;
3827
3828           /* speculatively enqueue b0 to the current next frame */
3829           bi0 = from[0];
3830           to_next[0] = bi0;
3831           from += 1;
3832           to_next += 1;
3833           n_left_from -= 1;
3834           n_left_to_next -= 1;
3835
3836           b0 = vlib_get_buffer (vm, bi0);
3837           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3838           ip0 = vlib_buffer_get_current (b0);
3839
3840           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3841
3842           vnet_buffer (b0)->snat.flags = 0;
3843           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3844             {
3845               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3846                 {
3847                   udp_header_t * udp0 = ip4_next_header (ip0);
3848                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3849
3850                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3851                 }
3852               else if (proto0 == SNAT_PROTOCOL_ICMP)
3853                 {
3854                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3855
3856                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3857                 }
3858               else
3859                 {
3860                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3861                 }
3862
3863               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3864             }
3865
3866           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3867
3868           /* verify speculative enqueue, maybe switch current next frame */
3869           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3870                                            to_next, n_left_to_next,
3871                                            bi0, next0);
3872          }
3873
3874       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3875     }
3876
3877   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3878                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3879                                pkts_processed);
3880   return frame->n_vectors;
3881 }
3882
3883 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3884   .function = snat_hairpin_dst_fn,
3885   .name = "nat44-hairpin-dst",
3886   .vector_size = sizeof (u32),
3887   .type = VLIB_NODE_TYPE_INTERNAL,
3888   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3889   .error_strings = snat_in2out_error_strings,
3890   .n_next_nodes = 2,
3891   .next_nodes = {
3892     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3893     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3894   },
3895 };
3896
3897 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3898                               snat_hairpin_dst_fn);
3899
3900 static uword
3901 snat_hairpin_src_fn (vlib_main_t * vm,
3902                      vlib_node_runtime_t * node,
3903                      vlib_frame_t * frame)
3904 {
3905   u32 n_left_from, * from, * to_next;
3906   snat_in2out_next_t next_index;
3907   u32 pkts_processed = 0;
3908   snat_main_t *sm = &snat_main;
3909
3910   from = vlib_frame_vector_args (frame);
3911   n_left_from = frame->n_vectors;
3912   next_index = node->cached_next_index;
3913
3914   while (n_left_from > 0)
3915     {
3916       u32 n_left_to_next;
3917
3918       vlib_get_next_frame (vm, node, next_index,
3919                            to_next, n_left_to_next);
3920
3921       while (n_left_from > 0 && n_left_to_next > 0)
3922         {
3923           u32 bi0;
3924           vlib_buffer_t * b0;
3925           u32 next0;
3926           snat_interface_t *i;
3927           u32 sw_if_index0;
3928
3929           /* speculatively enqueue b0 to the current next frame */
3930           bi0 = from[0];
3931           to_next[0] = bi0;
3932           from += 1;
3933           to_next += 1;
3934           n_left_from -= 1;
3935           n_left_to_next -= 1;
3936
3937           b0 = vlib_get_buffer (vm, bi0);
3938           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3939           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3940
3941           pool_foreach (i, sm->output_feature_interfaces,
3942           ({
3943             /* Only packets from NAT inside interface */
3944             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3945               {
3946                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3947                                     SNAT_FLAG_HAIRPINNING))
3948                   {
3949                     if (PREDICT_TRUE (sm->num_workers > 1))
3950                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3951                     else
3952                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3953                   }
3954                 break;
3955               }
3956           }));
3957
3958           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3959
3960           /* verify speculative enqueue, maybe switch current next frame */
3961           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3962                                            to_next, n_left_to_next,
3963                                            bi0, next0);
3964          }
3965
3966       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3967     }
3968
3969   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3970                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3971                                pkts_processed);
3972   return frame->n_vectors;
3973 }
3974
3975 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3976   .function = snat_hairpin_src_fn,
3977   .name = "nat44-hairpin-src",
3978   .vector_size = sizeof (u32),
3979   .type = VLIB_NODE_TYPE_INTERNAL,
3980   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3981   .error_strings = snat_in2out_error_strings,
3982   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3983   .next_nodes = {
3984      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3985      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3986      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3987      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3988   },
3989 };
3990
3991 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3992                               snat_hairpin_src_fn);
3993
3994 static uword
3995 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3996                                 vlib_node_runtime_t * node,
3997                                 vlib_frame_t * frame)
3998 {
3999   u32 n_left_from, * from, * to_next;
4000   snat_in2out_next_t next_index;
4001   u32 pkts_processed = 0;
4002   snat_main_t * sm = &snat_main;
4003   u32 stats_node_index;
4004
4005   stats_node_index = snat_in2out_fast_node.index;
4006
4007   from = vlib_frame_vector_args (frame);
4008   n_left_from = frame->n_vectors;
4009   next_index = node->cached_next_index;
4010
4011   while (n_left_from > 0)
4012     {
4013       u32 n_left_to_next;
4014
4015       vlib_get_next_frame (vm, node, next_index,
4016                            to_next, n_left_to_next);
4017
4018       while (n_left_from > 0 && n_left_to_next > 0)
4019         {
4020           u32 bi0;
4021           vlib_buffer_t * b0;
4022           u32 next0;
4023           u32 sw_if_index0;
4024           ip4_header_t * ip0;
4025           ip_csum_t sum0;
4026           u32 new_addr0, old_addr0;
4027           u16 old_port0, new_port0;
4028           udp_header_t * udp0;
4029           tcp_header_t * tcp0;
4030           icmp46_header_t * icmp0;
4031           snat_session_key_t key0, sm0;
4032           u32 proto0;
4033           u32 rx_fib_index0;
4034
4035           /* speculatively enqueue b0 to the current next frame */
4036           bi0 = from[0];
4037           to_next[0] = bi0;
4038           from += 1;
4039           to_next += 1;
4040           n_left_from -= 1;
4041           n_left_to_next -= 1;
4042
4043           b0 = vlib_get_buffer (vm, bi0);
4044           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4045
4046           ip0 = vlib_buffer_get_current (b0);
4047           udp0 = ip4_next_header (ip0);
4048           tcp0 = (tcp_header_t *) udp0;
4049           icmp0 = (icmp46_header_t *) udp0;
4050
4051           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4052           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4053
4054           if (PREDICT_FALSE(ip0->ttl == 1))
4055             {
4056               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4057               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4058                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4059                                            0);
4060               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4061               goto trace0;
4062             }
4063
4064           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4065
4066           if (PREDICT_FALSE (proto0 == ~0))
4067               goto trace0;
4068
4069           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4070             {
4071               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4072                                   rx_fib_index0, node, next0, ~0, 0, 0);
4073               goto trace0;
4074             }
4075
4076           key0.addr = ip0->src_address;
4077           key0.protocol = proto0;
4078           key0.port = udp0->src_port;
4079           key0.fib_index = rx_fib_index0;
4080
4081           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
4082             {
4083               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4084               next0= SNAT_IN2OUT_NEXT_DROP;
4085               goto trace0;
4086             }
4087
4088           new_addr0 = sm0.addr.as_u32;
4089           new_port0 = sm0.port;
4090           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4091           old_addr0 = ip0->src_address.as_u32;
4092           ip0->src_address.as_u32 = new_addr0;
4093
4094           sum0 = ip0->checksum;
4095           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4096                                  ip4_header_t,
4097                                  src_address /* changed member */);
4098           ip0->checksum = ip_csum_fold (sum0);
4099
4100           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4101             {
4102               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4103                 {
4104                   old_port0 = tcp0->src_port;
4105                   tcp0->src_port = new_port0;
4106
4107                   sum0 = tcp0->checksum;
4108                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4109                                          ip4_header_t,
4110                                          dst_address /* changed member */);
4111                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4112                                          ip4_header_t /* cheat */,
4113                                          length /* changed member */);
4114                   tcp0->checksum = ip_csum_fold(sum0);
4115                 }
4116               else
4117                 {
4118                   old_port0 = udp0->src_port;
4119                   udp0->src_port = new_port0;
4120                   udp0->checksum = 0;
4121                 }
4122             }
4123           else
4124             {
4125               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4126                 {
4127                   sum0 = tcp0->checksum;
4128                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4129                                          ip4_header_t,
4130                                          dst_address /* changed member */);
4131                   tcp0->checksum = ip_csum_fold(sum0);
4132                 }
4133             }
4134
4135           /* Hairpinning */
4136           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4137
4138         trace0:
4139           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4140                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4141             {
4142               snat_in2out_trace_t *t =
4143                  vlib_add_trace (vm, node, b0, sizeof (*t));
4144               t->sw_if_index = sw_if_index0;
4145               t->next_index = next0;
4146             }
4147
4148           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4149
4150           /* verify speculative enqueue, maybe switch current next frame */
4151           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4152                                            to_next, n_left_to_next,
4153                                            bi0, next0);
4154         }
4155
4156       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4157     }
4158
4159   vlib_node_increment_counter (vm, stats_node_index,
4160                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4161                                pkts_processed);
4162   return frame->n_vectors;
4163 }
4164
4165
4166 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
4167   .function = snat_in2out_fast_static_map_fn,
4168   .name = "nat44-in2out-fast",
4169   .vector_size = sizeof (u32),
4170   .format_trace = format_snat_in2out_fast_trace,
4171   .type = VLIB_NODE_TYPE_INTERNAL,
4172
4173   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4174   .error_strings = snat_in2out_error_strings,
4175
4176   .runtime_data_bytes = sizeof (snat_runtime_t),
4177
4178   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4179
4180   /* edit / add dispositions here */
4181   .next_nodes = {
4182     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4183     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4184     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4185     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4186     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4187   },
4188 };
4189
4190 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);