src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37   u32 is_slow_path;
  38 } snat_in2out_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_in2out_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  51   char * tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  66
  67   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  68               t->sw_if_index, t->next_index);
  69
  70   return s;
  71 }
  72
  73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   snat_in2out_worker_handoff_trace_t * t =
  78     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  79   char * m;
  80
  81   m = t->do_handoff ? "next worker" : "same worker";
  82   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  83
  84   return s;
  85 }
  86
  87 typedef struct {
  88   u32 sw_if_index;
  89   u32 next_index;
  90   u8 cached;
  91 } nat44_in2out_reass_trace_t;
  92
  93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  94 {
  95   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  96   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  97   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  98
  99   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 100               t->sw_if_index, t->next_index,
 101               t->cached ? "cached" : "translated");
 102
 103   return s;
 104 }
 105
 106 vlib_node_registration_t snat_in2out_node;
 107 vlib_node_registration_t snat_in2out_slowpath_node;
 108 vlib_node_registration_t snat_in2out_fast_node;
 109 vlib_node_registration_t snat_in2out_worker_handoff_node;
 110 vlib_node_registration_t snat_det_in2out_node;
 111 vlib_node_registration_t snat_in2out_output_node;
 112 vlib_node_registration_t snat_in2out_output_slowpath_node;
 113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 114 vlib_node_registration_t snat_hairpin_dst_node;
 115 vlib_node_registration_t snat_hairpin_src_node;
 116 vlib_node_registration_t nat44_hairpinning_node;
 117 vlib_node_registration_t nat44_in2out_reass_node;
 118
 119
 120 #define foreach_snat_in2out_error                       \
 121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 122 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 123 _(OUT_OF_PORTS, "Out of ports")                         \
 124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 125 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 126 _(NO_TRANSLATION, "No translation")                     \
 127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 128 _(DROP_FRAGMENT, "Drop fragment")                       \
 129 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 131
 132 typedef enum {
 133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 134   foreach_snat_in2out_error
 135 #undef _
 136   SNAT_IN2OUT_N_ERROR,
 137 } snat_in2out_error_t;
 138
 139 static char * snat_in2out_error_strings[] = {
 140 #define _(sym,string) string,
 141   foreach_snat_in2out_error
 142 #undef _
 143 };
 144
 145 typedef enum {
 146   SNAT_IN2OUT_NEXT_LOOKUP,
 147   SNAT_IN2OUT_NEXT_DROP,
 148   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 149   SNAT_IN2OUT_NEXT_SLOW_PATH,
 150   SNAT_IN2OUT_NEXT_REASS,
 151   SNAT_IN2OUT_N_NEXT,
 152 } snat_in2out_next_t;
 153
 154 typedef enum {
 155   SNAT_HAIRPIN_SRC_NEXT_DROP,
 156   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 157   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 158   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 159   SNAT_HAIRPIN_SRC_N_NEXT,
 160 } snat_hairpin_next_t;
 161
 162 /**
 163  * @brief Check if packet should be translated
 164  *
 165  * Packets aimed at outside interface and external address with active session
 166  * should be translated.
 167  *
 168  * @param sm            NAT main
 169  * @param rt            NAT runtime data
 170  * @param sw_if_index0  index of the inside interface
 171  * @param ip0           IPv4 header
 172  * @param proto0        NAT protocol
 173  * @param rx_fib_index0 RX FIB index
 174  *
 175  * @returns 0 if packet should be translated otherwise 1
 176  */
 177 static inline int
 178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 179                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                          u32 rx_fib_index0)
 181 {
 182   if (sm->out2in_dpo)
 183     return 0;
 184
 185   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 186   fib_prefix_t pfx = {
 187     .fp_proto = FIB_PROTOCOL_IP4,
 188     .fp_len = 32,
 189     .fp_addr = {
 190         .ip4.as_u32 = ip0->dst_address.as_u32,
 191     },
 192   };
 193
 194   /* Don't NAT packet aimed at the intfc address */
 195   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 196                                       ip0->dst_address.as_u32)))
 197     return 1;
 198
 199   fei = fib_table_lookup (rx_fib_index0, &pfx);
 200   if (FIB_NODE_INDEX_INVALID != fei)
 201     {
 202       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 203       if (sw_if_index == ~0)
 204         {
 205           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 206           if (FIB_NODE_INDEX_INVALID != fei)
 207             sw_if_index = fib_entry_get_resolving_interface (fei);
 208         }
 209       snat_interface_t *i;
 210       pool_foreach (i, sm->interfaces,
 211       ({
 212         /* NAT packet aimed at outside interface */
 213         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 214           return 0;
 215       }));
 216     }
 217
 218   return 1;
 219 }
 220
 221 static inline int
 222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 223                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 224                     u32 rx_fib_index0, u32 thread_index)
 225 {
 226   udp_header_t * udp0 = ip4_next_header (ip0);
 227   snat_session_key_t key0, sm0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229
 230   key0.addr = ip0->dst_address;
 231   key0.port = udp0->dst_port;
 232   key0.protocol = proto0;
 233   key0.fib_index = sm->outside_fib_index;
 234   kv0.key = key0.as_u64;
 235
 236   /* NAT packet aimed at external address if */
 237   /* has active sessions */
 238   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 239                               &value0))
 240     {
 241       /* or is static mappings */
 242       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
 243         return 0;
 244     }
 245   else
 246     return 0;
 247
 248   if (sm->forwarding_enabled)
 249     return 1;
 250
 251   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 252                                  rx_fib_index0);
 253 }
 254
 255 static inline int
 256 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 257                                   u32 proto0, u32 thread_index)
 258 {
 259   udp_header_t * udp0 = ip4_next_header (ip0);
 260   snat_session_key_t key0;
 261   clib_bihash_kv_8_8_t kv0, value0;
 262
 263   key0.addr = ip0->src_address;
 264   key0.port = udp0->src_port;
 265   key0.protocol = proto0;
 266   key0.fib_index = sm->outside_fib_index;
 267   kv0.key = key0.as_u64;
 268
 269   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 270                               &value0))
 271     return 1;
 272
 273   return 0;
 274 }
 275
 276 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 277                       ip4_header_t * ip0,
 278                       u32 rx_fib_index0,
 279                       snat_session_key_t * key0,
 280                       snat_session_t ** sessionp,
 281                       vlib_node_runtime_t * node,
 282                       u32 next0,
 283                       u32 thread_index)
 284 {
 285   snat_user_t *u;
 286   snat_session_t *s;
 287   clib_bihash_kv_8_8_t kv0;
 288   snat_session_key_t key1;
 289   u32 address_index = ~0;
 290   u32 outside_fib_index;
 291   uword * p;
 292   udp_header_t * udp0 = ip4_next_header (ip0);
 293
 294   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 295     {
 296       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 297       nat_ipfix_logging_max_sessions(sm->max_translations);
 298       return SNAT_IN2OUT_NEXT_DROP;
 299     }
 300
 301   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 302   if (! p)
 303     {
 304       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 305       return SNAT_IN2OUT_NEXT_DROP;
 306     }
 307   outside_fib_index = p[0];
 308
 309   key1.protocol = key0->protocol;
 310
 311   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 312                               thread_index);
 313   if (!u)
 314     {
 315       clib_warning ("create NAT user failed");
 316       return SNAT_IN2OUT_NEXT_DROP;
 317     }
 318
 319   /* First try to match static mapping by local address and port */
 320   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0))
 321     {
 322       /* Try to create dynamic translation */
 323       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 324                                                thread_index, &key1,
 325                                                &address_index,
 326                                                sm->port_per_thread,
 327                                                sm->per_thread_data[thread_index].snat_thread_index))
 328         {
 329           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 330           return SNAT_IN2OUT_NEXT_DROP;
 331         }
 332       u->nsessions++;
 333     }
 334   else
 335     {
 336       u->nstaticsessions++;
 337     }
 338
 339   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 340   if (!s)
 341     {
 342       clib_warning ("create NAT session failed");
 343       return SNAT_IN2OUT_NEXT_DROP;
 344     }
 345
 346   if (address_index == ~0)
 347     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 348   s->outside_address_index = address_index;
 349   s->in2out = *key0;
 350   s->out2in = key1;
 351   s->out2in.protocol = key0->protocol;
 352   s->out2in.fib_index = outside_fib_index;
 353   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 354   s->ext_host_port = udp0->dst_port;
 355   *sessionp = s;
 356
 357   /* Add to translation hashes */
 358   kv0.key = s->in2out.as_u64;
 359   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 360   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 361                                1 /* is_add */))
 362       clib_warning ("in2out key add failed");
 363
 364   kv0.key = s->out2in.as_u64;
 365   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 366
 367   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 368                                1 /* is_add */))
 369       clib_warning ("out2in key add failed");
 370
 371   /* log NAT event */
 372   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 373                                       s->out2in.addr.as_u32,
 374                                       s->in2out.protocol,
 375                                       s->in2out.port,
 376                                       s->out2in.port,
 377                                       s->in2out.fib_index);
 378   return next0;
 379 }
 380
 381 static_always_inline
 382 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 383                                  snat_session_key_t *p_key0)
 384 {
 385   icmp46_header_t *icmp0;
 386   snat_session_key_t key0;
 387   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 388   ip4_header_t *inner_ip0 = 0;
 389   void *l4_header = 0;
 390   icmp46_header_t *inner_icmp0;
 391
 392   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 393   echo0 = (icmp_echo_header_t *)(icmp0+1);
 394
 395   if (!icmp_is_error_message (icmp0))
 396     {
 397       key0.protocol = SNAT_PROTOCOL_ICMP;
 398       key0.addr = ip0->src_address;
 399       key0.port = echo0->identifier;
 400     }
 401   else
 402     {
 403       inner_ip0 = (ip4_header_t *)(echo0+1);
 404       l4_header = ip4_next_header (inner_ip0);
 405       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 406       key0.addr = inner_ip0->dst_address;
 407       switch (key0.protocol)
 408         {
 409         case SNAT_PROTOCOL_ICMP:
 410           inner_icmp0 = (icmp46_header_t*)l4_header;
 411           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 412           key0.port = inner_echo0->identifier;
 413           break;
 414         case SNAT_PROTOCOL_UDP:
 415         case SNAT_PROTOCOL_TCP:
 416           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 417           break;
 418         default:
 419           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 420         }
 421     }
 422   *p_key0 = key0;
 423   return -1; /* success */
 424 }
 425
 426 static_always_inline int
 427 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 428 {
 429   icmp46_header_t *icmp0;
 430   nat_ed_ses_key_t key0;
 431   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 432   ip4_header_t *inner_ip0 = 0;
 433   void *l4_header = 0;
 434   icmp46_header_t *inner_icmp0;
 435
 436   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 437   echo0 = (icmp_echo_header_t *)(icmp0+1);
 438
 439   if (!icmp_is_error_message (icmp0))
 440     {
 441       key0.proto = IP_PROTOCOL_ICMP;
 442       key0.l_addr = ip0->src_address;
 443       key0.r_addr = ip0->dst_address;
 444       key0.l_port = key0.r_port = echo0->identifier;
 445     }
 446   else
 447     {
 448       inner_ip0 = (ip4_header_t *)(echo0+1);
 449       l4_header = ip4_next_header (inner_ip0);
 450       key0.proto = inner_ip0->protocol;
 451       key0.r_addr = inner_ip0->src_address;
 452       key0.l_addr = inner_ip0->dst_address;
 453       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 454         {
 455         case SNAT_PROTOCOL_ICMP:
 456           inner_icmp0 = (icmp46_header_t*)l4_header;
 457           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 458           key0.r_port = key0.l_port = inner_echo0->identifier;
 459           break;
 460         case SNAT_PROTOCOL_UDP:
 461         case SNAT_PROTOCOL_TCP:
 462           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 463           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
 464           break;
 465         default:
 466           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 467         }
 468     }
 469   *p_key0 = key0;
 470   return 0;
 471 }
 472
 473 static inline int
 474 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip)
 475 {
 476   nat_ed_ses_key_t key;
 477   clib_bihash_kv_16_8_t kv, value;
 478   udp_header_t *udp;
 479
 480   if (!sm->forwarding_enabled)
 481     return 0;
 482
 483   if (ip->protocol == IP_PROTOCOL_ICMP)
 484     {
 485       if (icmp_get_ed_key (ip, &key))
 486         return 0;
 487     }
 488   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 489     {
 490       udp = ip4_next_header(ip);
 491       key.l_addr = ip->src_address;
 492       key.r_addr = ip->dst_address;
 493       key.proto = ip->protocol;
 494       key.r_port = udp->dst_port;
 495       key.l_port = udp->src_port;
 496     }
 497   else
 498     {
 499       key.l_addr = ip->src_address;
 500       key.r_addr = ip->dst_address;
 501       key.proto = ip->protocol;
 502       key.l_port = key.r_port = 0;
 503     }
 504   key.fib_index = 0;
 505   kv.key[0] = key.as_u64[0];
 506   kv.key[1] = key.as_u64[1];
 507
 508   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 509     return value.value == ~0ULL;
 510
 511   return 0;
 512 }
 513
 514 /**
 515  * Get address and port values to be used for ICMP packet translation
 516  * and create session if needed
 517  *
 518  * @param[in,out] sm             NAT main
 519  * @param[in,out] node           NAT node runtime
 520  * @param[in] thread_index       thread index
 521  * @param[in,out] b0             buffer containing packet to be translated
 522  * @param[out] p_proto           protocol used for matching
 523  * @param[out] p_value           address and port after NAT translation
 524  * @param[out] p_dont_translate  if packet should not be translated
 525  * @param d                      optional parameter
 526  * @param e                      optional parameter
 527  */
 528 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 529                            u32 thread_index, vlib_buffer_t *b0,
 530                            ip4_header_t *ip0, u8 *p_proto,
 531                            snat_session_key_t *p_value,
 532                            u8 *p_dont_translate, void *d, void *e)
 533 {
 534   icmp46_header_t *icmp0;
 535   u32 sw_if_index0;
 536   u32 rx_fib_index0;
 537   snat_session_key_t key0;
 538   snat_session_t *s0 = 0;
 539   u8 dont_translate = 0;
 540   clib_bihash_kv_8_8_t kv0, value0;
 541   u32 next0 = ~0;
 542   int err;
 543
 544   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 545   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 546   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 547
 548   err = icmp_get_key (ip0, &key0);
 549   if (err != -1)
 550     {
 551       b0->error = node->errors[err];
 552       next0 = SNAT_IN2OUT_NEXT_DROP;
 553       goto out;
 554     }
 555   key0.fib_index = rx_fib_index0;
 556
 557   kv0.key = key0.as_u64;
 558
 559   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 560                               &value0))
 561     {
 562       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 563         {
 564           if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
 565               ip0, IP_PROTOCOL_ICMP, thread_index)))
 566             {
 567               dont_translate = 1;
 568               goto out;
 569             }
 570         }
 571       else
 572         {
 573           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 574               ip0, IP_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 575             {
 576               dont_translate = 1;
 577               goto out;
 578             }
 579         }
 580
 581       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 582         {
 583           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 584           next0 = SNAT_IN2OUT_NEXT_DROP;
 585           goto out;
 586         }
 587
 588       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 589                          &s0, node, next0, thread_index);
 590
 591       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 592         goto out;
 593     }
 594   else
 595     {
 596       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 597                         icmp0->type != ICMP4_echo_reply &&
 598                         !icmp_is_error_message (icmp0)))
 599         {
 600           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 601           next0 = SNAT_IN2OUT_NEXT_DROP;
 602           goto out;
 603         }
 604
 605       if (PREDICT_FALSE (value0.value == ~0ULL))
 606         {
 607           nat_ed_ses_key_t key;
 608           clib_bihash_kv_16_8_t s_kv, s_value;
 609
 610           key.as_u64[0] = 0;
 611           key.as_u64[1] = 0;
 612           if (icmp_get_ed_key (ip0, &key))
 613             {
 614               b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 615               next0 = SNAT_IN2OUT_NEXT_DROP;
 616               goto out;
 617             }
 618           key.fib_index = rx_fib_index0;
 619           s_kv.key[0] = key.as_u64[0];
 620           s_kv.key[1] = key.as_u64[1];
 621           if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 622             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 623                                     s_value.value);
 624           else
 625            {
 626               next0 = SNAT_IN2OUT_NEXT_DROP;
 627               goto out;
 628            }
 629         }
 630       else
 631         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 632                                 value0.value);
 633     }
 634
 635 out:
 636   *p_proto = key0.protocol;
 637   if (s0)
 638     *p_value = s0->out2in;
 639   *p_dont_translate = dont_translate;
 640   if (d)
 641     *(snat_session_t**)d = s0;
 642   return next0;
 643 }
 644
 645 /**
 646  * Get address and port values to be used for ICMP packet translation
 647  *
 648  * @param[in] sm                 NAT main
 649  * @param[in,out] node           NAT node runtime
 650  * @param[in] thread_index       thread index
 651  * @param[in,out] b0             buffer containing packet to be translated
 652  * @param[out] p_proto           protocol used for matching
 653  * @param[out] p_value           address and port after NAT translation
 654  * @param[out] p_dont_translate  if packet should not be translated
 655  * @param d                      optional parameter
 656  * @param e                      optional parameter
 657  */
 658 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 659                            u32 thread_index, vlib_buffer_t *b0,
 660                            ip4_header_t *ip0, u8 *p_proto,
 661                            snat_session_key_t *p_value,
 662                            u8 *p_dont_translate, void *d, void *e)
 663 {
 664   icmp46_header_t *icmp0;
 665   u32 sw_if_index0;
 666   u32 rx_fib_index0;
 667   snat_session_key_t key0;
 668   snat_session_key_t sm0;
 669   u8 dont_translate = 0;
 670   u8 is_addr_only;
 671   u32 next0 = ~0;
 672   int err;
 673
 674   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 675   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 676   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 677
 678   err = icmp_get_key (ip0, &key0);
 679   if (err != -1)
 680     {
 681       b0->error = node->errors[err];
 682       next0 = SNAT_IN2OUT_NEXT_DROP;
 683       goto out2;
 684     }
 685   key0.fib_index = rx_fib_index0;
 686
 687   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0))
 688     {
 689       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 690           IP_PROTOCOL_ICMP, rx_fib_index0)))
 691         {
 692           dont_translate = 1;
 693           goto out;
 694         }
 695
 696       if (icmp_is_error_message (icmp0))
 697         {
 698           next0 = SNAT_IN2OUT_NEXT_DROP;
 699           goto out;
 700         }
 701
 702       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 703       next0 = SNAT_IN2OUT_NEXT_DROP;
 704       goto out;
 705     }
 706
 707   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 708                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 709                     !icmp_is_error_message (icmp0)))
 710     {
 711       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 712       next0 = SNAT_IN2OUT_NEXT_DROP;
 713       goto out;
 714     }
 715
 716 out:
 717   *p_value = sm0;
 718 out2:
 719   *p_proto = key0.protocol;
 720   *p_dont_translate = dont_translate;
 721   return next0;
 722 }
 723
 724 static inline u32 icmp_in2out (snat_main_t *sm,
 725                                vlib_buffer_t * b0,
 726                                ip4_header_t * ip0,
 727                                icmp46_header_t * icmp0,
 728                                u32 sw_if_index0,
 729                                u32 rx_fib_index0,
 730                                vlib_node_runtime_t * node,
 731                                u32 next0,
 732                                u32 thread_index,
 733                                void *d,
 734                                void *e)
 735 {
 736   snat_session_key_t sm0;
 737   u8 protocol;
 738   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 739   ip4_header_t *inner_ip0;
 740   void *l4_header = 0;
 741   icmp46_header_t *inner_icmp0;
 742   u8 dont_translate;
 743   u32 new_addr0, old_addr0;
 744   u16 old_id0, new_id0;
 745   ip_csum_t sum0;
 746   u16 checksum0;
 747   u32 next0_tmp;
 748
 749   echo0 = (icmp_echo_header_t *)(icmp0+1);
 750
 751   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 752                                        &protocol, &sm0, &dont_translate, d, e);
 753   if (next0_tmp != ~0)
 754     next0 = next0_tmp;
 755   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 756     goto out;
 757
 758   sum0 = ip_incremental_checksum (0, icmp0,
 759                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 760   checksum0 = ~ip_csum_fold (sum0);
 761   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 762     {
 763       next0 = SNAT_IN2OUT_NEXT_DROP;
 764       goto out;
 765     }
 766
 767   old_addr0 = ip0->src_address.as_u32;
 768   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 769   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 770     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 771
 772   sum0 = ip0->checksum;
 773   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 774                          src_address /* changed member */);
 775   ip0->checksum = ip_csum_fold (sum0);
 776
 777   if (!icmp_is_error_message (icmp0))
 778     {
 779       new_id0 = sm0.port;
 780       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 781         {
 782           old_id0 = echo0->identifier;
 783           new_id0 = sm0.port;
 784           echo0->identifier = new_id0;
 785
 786           sum0 = icmp0->checksum;
 787           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 788                                  identifier);
 789           icmp0->checksum = ip_csum_fold (sum0);
 790         }
 791     }
 792   else
 793     {
 794       inner_ip0 = (ip4_header_t *)(echo0+1);
 795       l4_header = ip4_next_header (inner_ip0);
 796
 797       if (!ip4_header_checksum_is_valid (inner_ip0))
 798         {
 799           next0 = SNAT_IN2OUT_NEXT_DROP;
 800           goto out;
 801         }
 802
 803       old_addr0 = inner_ip0->dst_address.as_u32;
 804       inner_ip0->dst_address = sm0.addr;
 805       new_addr0 = inner_ip0->dst_address.as_u32;
 806
 807       sum0 = icmp0->checksum;
 808       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 809                              dst_address /* changed member */);
 810       icmp0->checksum = ip_csum_fold (sum0);
 811
 812       switch (protocol)
 813         {
 814           case SNAT_PROTOCOL_ICMP:
 815             inner_icmp0 = (icmp46_header_t*)l4_header;
 816             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 817
 818             old_id0 = inner_echo0->identifier;
 819             new_id0 = sm0.port;
 820             inner_echo0->identifier = new_id0;
 821
 822             sum0 = icmp0->checksum;
 823             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 824                                    identifier);
 825             icmp0->checksum = ip_csum_fold (sum0);
 826             break;
 827           case SNAT_PROTOCOL_UDP:
 828           case SNAT_PROTOCOL_TCP:
 829             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 830             new_id0 = sm0.port;
 831             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 832
 833             sum0 = icmp0->checksum;
 834             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 835                                    dst_port);
 836             icmp0->checksum = ip_csum_fold (sum0);
 837             break;
 838           default:
 839             ASSERT(0);
 840         }
 841     }
 842
 843 out:
 844   return next0;
 845 }
 846
 847 /**
 848  * @brief Hairpinning
 849  *
 850  * Hairpinning allows two endpoints on the internal side of the NAT to
 851  * communicate even if they only use each other's external IP addresses
 852  * and ports.
 853  *
 854  * @param sm     NAT main.
 855  * @param b0     Vlib buffer.
 856  * @param ip0    IP header.
 857  * @param udp0   UDP header.
 858  * @param tcp0   TCP header.
 859  * @param proto0 NAT protocol.
 860  */
 861 static inline int
 862 snat_hairpinning (snat_main_t *sm,
 863                   vlib_buffer_t * b0,
 864                   ip4_header_t * ip0,
 865                   udp_header_t * udp0,
 866                   tcp_header_t * tcp0,
 867                   u32 proto0)
 868 {
 869   snat_session_key_t key0, sm0;
 870   snat_session_t * s0;
 871   clib_bihash_kv_8_8_t kv0, value0;
 872   ip_csum_t sum0;
 873   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 874   u16 new_dst_port0, old_dst_port0;
 875
 876   key0.addr = ip0->dst_address;
 877   key0.port = udp0->dst_port;
 878   key0.protocol = proto0;
 879   key0.fib_index = sm->outside_fib_index;
 880   kv0.key = key0.as_u64;
 881
 882   /* Check if destination is static mappings */
 883   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
 884     {
 885       new_dst_addr0 = sm0.addr.as_u32;
 886       new_dst_port0 = sm0.port;
 887       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 888     }
 889   /* or active session */
 890   else
 891     {
 892       if (sm->num_workers > 1)
 893         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 894       else
 895         ti = sm->num_workers;
 896
 897       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 898         {
 899           si = value0.value;
 900
 901           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 902           new_dst_addr0 = s0->in2out.addr.as_u32;
 903           new_dst_port0 = s0->in2out.port;
 904           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 905         }
 906     }
 907
 908   /* Destination is behind the same NAT, use internal address and port */
 909   if (new_dst_addr0)
 910     {
 911       old_dst_addr0 = ip0->dst_address.as_u32;
 912       ip0->dst_address.as_u32 = new_dst_addr0;
 913       sum0 = ip0->checksum;
 914       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 915                              ip4_header_t, dst_address);
 916       ip0->checksum = ip_csum_fold (sum0);
 917
 918       old_dst_port0 = tcp0->dst;
 919       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 920         {
 921           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 922             {
 923               tcp0->dst = new_dst_port0;
 924               sum0 = tcp0->checksum;
 925               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 926                                      ip4_header_t, dst_address);
 927               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 928                                      ip4_header_t /* cheat */, length);
 929               tcp0->checksum = ip_csum_fold(sum0);
 930             }
 931           else
 932             {
 933               udp0->dst_port = new_dst_port0;
 934               udp0->checksum = 0;
 935             }
 936         }
 937       else
 938         {
 939           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 940             {
 941               sum0 = tcp0->checksum;
 942               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 943                                      ip4_header_t, dst_address);
 944               tcp0->checksum = ip_csum_fold(sum0);
 945             }
 946         }
 947       return 1;
 948     }
 949   return 0;
 950 }
 951
 952 static inline void
 953 snat_icmp_hairpinning (snat_main_t *sm,
 954                        vlib_buffer_t * b0,
 955                        ip4_header_t * ip0,
 956                        icmp46_header_t * icmp0)
 957 {
 958   snat_session_key_t key0, sm0;
 959   clib_bihash_kv_8_8_t kv0, value0;
 960   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 961   ip_csum_t sum0;
 962   snat_session_t *s0;
 963
 964   if (!icmp_is_error_message (icmp0))
 965     {
 966       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
 967       u16 icmp_id0 = echo0->identifier;
 968       key0.addr = ip0->dst_address;
 969       key0.port = icmp_id0;
 970       key0.protocol = SNAT_PROTOCOL_ICMP;
 971       key0.fib_index = sm->outside_fib_index;
 972       kv0.key = key0.as_u64;
 973
 974       if (sm->num_workers > 1)
 975         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
 976       else
 977         ti = sm->num_workers;
 978
 979       /* Check if destination is in active sessions */
 980       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
 981                                   &value0))
 982         {
 983           /* or static mappings */
 984           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
 985             {
 986               new_dst_addr0 = sm0.addr.as_u32;
 987               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 988             }
 989         }
 990       else
 991         {
 992           si = value0.value;
 993
 994           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 995           new_dst_addr0 = s0->in2out.addr.as_u32;
 996           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 997           echo0->identifier = s0->in2out.port;
 998           sum0 = icmp0->checksum;
 999           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1000                                  icmp_echo_header_t, identifier);
1001           icmp0->checksum = ip_csum_fold (sum0);
1002         }
1003
1004       /* Destination is behind the same NAT, use internal address and port */
1005       if (new_dst_addr0)
1006         {
1007           old_dst_addr0 = ip0->dst_address.as_u32;
1008           ip0->dst_address.as_u32 = new_dst_addr0;
1009           sum0 = ip0->checksum;
1010           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1011                                  ip4_header_t, dst_address);
1012           ip0->checksum = ip_csum_fold (sum0);
1013         }
1014     }
1015
1016 }
1017
1018 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1019                                          vlib_buffer_t * b0,
1020                                          ip4_header_t * ip0,
1021                                          icmp46_header_t * icmp0,
1022                                          u32 sw_if_index0,
1023                                          u32 rx_fib_index0,
1024                                          vlib_node_runtime_t * node,
1025                                          u32 next0,
1026                                          f64 now,
1027                                          u32 thread_index,
1028                                          snat_session_t ** p_s0)
1029 {
1030   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1031                       next0, thread_index, p_s0, 0);
1032   snat_session_t * s0 = *p_s0;
1033   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1034     {
1035       /* Hairpinning */
1036       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1037         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1038       /* Accounting */
1039       s0->last_heard = now;
1040       s0->total_pkts++;
1041       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1042       /* Per-user LRU list maintenance */
1043       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1044                          s0->per_user_index);
1045       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1046                           s0->per_user_list_head_index,
1047                           s0->per_user_index);
1048     }
1049   return next0;
1050 }
1051 static inline void
1052 snat_hairpinning_unknown_proto (snat_main_t *sm,
1053                                 vlib_buffer_t * b,
1054                                 ip4_header_t * ip)
1055 {
1056   u32 old_addr, new_addr = 0, ti = 0;
1057   clib_bihash_kv_8_8_t kv, value;
1058   clib_bihash_kv_16_8_t s_kv, s_value;
1059   nat_ed_ses_key_t key;
1060   snat_session_key_t m_key;
1061   snat_static_mapping_t *m;
1062   ip_csum_t sum;
1063   snat_session_t *s;
1064
1065   old_addr = ip->dst_address.as_u32;
1066   key.l_addr.as_u32 = ip->dst_address.as_u32;
1067   key.r_addr.as_u32 = ip->src_address.as_u32;
1068   key.fib_index = sm->outside_fib_index;
1069   key.proto = ip->protocol;
1070   key.r_port = 0;
1071   key.l_port = 0;
1072   s_kv.key[0] = key.as_u64[0];
1073   s_kv.key[1] = key.as_u64[1];
1074   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1075     {
1076       m_key.addr = ip->dst_address;
1077       m_key.fib_index = sm->outside_fib_index;
1078       m_key.port = 0;
1079       m_key.protocol = 0;
1080       kv.key = m_key.as_u64;
1081       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1082         return;
1083
1084       m = pool_elt_at_index (sm->static_mappings, value.value);
1085       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1086         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1087       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1088     }
1089   else
1090     {
1091       if (sm->num_workers > 1)
1092         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1093       else
1094         ti = sm->num_workers;
1095
1096       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1097       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1098         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1099       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1100     }
1101   sum = ip->checksum;
1102   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1103   ip->checksum = ip_csum_fold (sum);
1104 }
1105
1106 static snat_session_t *
1107 snat_in2out_unknown_proto (snat_main_t *sm,
1108                            vlib_buffer_t * b,
1109                            ip4_header_t * ip,
1110                            u32 rx_fib_index,
1111                            u32 thread_index,
1112                            f64 now,
1113                            vlib_main_t * vm,
1114                            vlib_node_runtime_t * node)
1115 {
1116   clib_bihash_kv_8_8_t kv, value;
1117   clib_bihash_kv_16_8_t s_kv, s_value;
1118   snat_static_mapping_t *m;
1119   snat_session_key_t m_key;
1120   u32 old_addr, new_addr = 0;
1121   ip_csum_t sum;
1122   snat_user_t *u;
1123   dlist_elt_t *head, *elt;
1124   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1125   u32 elt_index, head_index, ses_index;
1126   snat_session_t * s;
1127   nat_ed_ses_key_t key;
1128   u32 address_index = ~0;
1129   int i;
1130   u8 is_sm = 0;
1131
1132   old_addr = ip->src_address.as_u32;
1133
1134   key.l_addr = ip->src_address;
1135   key.r_addr = ip->dst_address;
1136   key.fib_index = rx_fib_index;
1137   key.proto = ip->protocol;
1138   key.l_port = 0;
1139   key.l_port = 0;
1140   s_kv.key[0] = key.as_u64[0];
1141   s_kv.key[1] = key.as_u64[1];
1142
1143   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1144     {
1145       s = pool_elt_at_index (tsm->sessions, s_value.value);
1146       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1147     }
1148   else
1149     {
1150       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1151         {
1152           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1153           nat_ipfix_logging_max_sessions(sm->max_translations);
1154           return 0;
1155         }
1156
1157       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1158                                   thread_index);
1159       if (!u)
1160         {
1161           clib_warning ("create NAT user failed");
1162           return 0;
1163         }
1164
1165       m_key.addr = ip->src_address;
1166       m_key.port = 0;
1167       m_key.protocol = 0;
1168       m_key.fib_index = rx_fib_index;
1169       kv.key = m_key.as_u64;
1170
1171       /* Try to find static mapping first */
1172       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1173         {
1174           m = pool_elt_at_index (sm->static_mappings, value.value);
1175           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1176           is_sm = 1;
1177           goto create_ses;
1178         }
1179       /* Fallback to 3-tuple key */
1180       else
1181         {
1182           /* Choose same out address as for TCP/UDP session to same destination */
1183           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1184             {
1185               head_index = u->sessions_per_user_list_head_index;
1186               head = pool_elt_at_index (tsm->list_pool, head_index);
1187               elt_index = head->next;
1188               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1189               ses_index = elt->value;
1190               while (ses_index != ~0)
1191                 {
1192                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1193                   elt_index = elt->next;
1194                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1195                   ses_index = elt->value;
1196
1197                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1198                     {
1199                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1200                       address_index = s->outside_address_index;
1201
1202                       key.fib_index = sm->outside_fib_index;
1203                       key.l_addr.as_u32 = new_addr;
1204                       s_kv.key[0] = key.as_u64[0];
1205                       s_kv.key[1] = key.as_u64[1];
1206                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1207                         break;
1208
1209                       goto create_ses;
1210                     }
1211                 }
1212             }
1213           key.fib_index = sm->outside_fib_index;
1214           for (i = 0; i < vec_len (sm->addresses); i++)
1215             {
1216               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1217               s_kv.key[0] = key.as_u64[0];
1218               s_kv.key[1] = key.as_u64[1];
1219               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1220                 {
1221                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1222                   address_index = i;
1223                   goto create_ses;
1224                 }
1225             }
1226           return 0;
1227         }
1228
1229 create_ses:
1230       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1231       if (!s)
1232         {
1233           clib_warning ("create NAT session failed");
1234           return 0;
1235         }
1236
1237       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1238       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1239       s->outside_address_index = address_index;
1240       s->out2in.addr.as_u32 = new_addr;
1241       s->out2in.fib_index = sm->outside_fib_index;
1242       s->in2out.addr.as_u32 = old_addr;
1243       s->in2out.fib_index = rx_fib_index;
1244       s->in2out.port = s->out2in.port = ip->protocol;
1245       if (is_sm)
1246         {
1247           u->nstaticsessions++;
1248           s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1249         }
1250       else
1251         {
1252           u->nsessions++;
1253         }
1254
1255       /* Add to lookup tables */
1256       key.l_addr.as_u32 = old_addr;
1257       key.r_addr = ip->dst_address;
1258       key.proto = ip->protocol;
1259       key.fib_index = rx_fib_index;
1260       s_kv.key[0] = key.as_u64[0];
1261       s_kv.key[1] = key.as_u64[1];
1262       s_kv.value = s - tsm->sessions;
1263       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1264         clib_warning ("in2out key add failed");
1265
1266       key.l_addr.as_u32 = new_addr;
1267       key.fib_index = sm->outside_fib_index;
1268       s_kv.key[0] = key.as_u64[0];
1269       s_kv.key[1] = key.as_u64[1];
1270       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1271         clib_warning ("out2in key add failed");
1272   }
1273
1274   /* Update IP checksum */
1275   sum = ip->checksum;
1276   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1277   ip->checksum = ip_csum_fold (sum);
1278
1279   /* Accounting */
1280   s->last_heard = now;
1281   s->total_pkts++;
1282   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1283   /* Per-user LRU list maintenance */
1284   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1285   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1286                       s->per_user_index);
1287
1288   /* Hairpinning */
1289   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1290     snat_hairpinning_unknown_proto(sm, b, ip);
1291
1292   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1293     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1294
1295   return s;
1296 }
1297
1298 static snat_session_t *
1299 snat_in2out_lb (snat_main_t *sm,
1300                 vlib_buffer_t * b,
1301                 ip4_header_t * ip,
1302                 u32 rx_fib_index,
1303                 u32 thread_index,
1304                 f64 now,
1305                 vlib_main_t * vm,
1306                 vlib_node_runtime_t * node)
1307 {
1308   nat_ed_ses_key_t key;
1309   clib_bihash_kv_16_8_t s_kv, s_value;
1310   udp_header_t *udp = ip4_next_header (ip);
1311   tcp_header_t *tcp = (tcp_header_t *) udp;
1312   snat_session_t *s = 0;
1313   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1314   u32 old_addr, new_addr;
1315   u16 new_port, old_port;
1316   ip_csum_t sum;
1317   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1318   snat_session_key_t e_key, l_key;
1319   snat_user_t *u;
1320
1321   old_addr = ip->src_address.as_u32;
1322
1323   key.l_addr = ip->src_address;
1324   key.r_addr = ip->dst_address;
1325   key.fib_index = rx_fib_index;
1326   key.proto = ip->protocol;
1327   key.r_port = udp->dst_port;
1328   key.l_port = udp->src_port;
1329   s_kv.key[0] = key.as_u64[0];
1330   s_kv.key[1] = key.as_u64[1];
1331
1332   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1333     {
1334       if (s_value.value == ~0ULL)
1335         return 0;
1336       s = pool_elt_at_index (tsm->sessions, s_value.value);
1337     }
1338   else
1339     {
1340       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1341         {
1342           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1343           nat_ipfix_logging_max_sessions(sm->max_translations);
1344           return 0;
1345         }
1346
1347       l_key.addr = ip->src_address;
1348       l_key.port = udp->src_port;
1349       l_key.protocol = proto;
1350       l_key.fib_index = rx_fib_index;
1351       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0))
1352         return 0;
1353
1354       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1355                                   thread_index);
1356       if (!u)
1357         {
1358           clib_warning ("create NAT user failed");
1359           return 0;
1360         }
1361
1362       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1363       if (!s)
1364         {
1365           clib_warning ("create NAT session failed");
1366           return 0;
1367         }
1368
1369       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1370       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1371       s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1372       s->outside_address_index = ~0;
1373       s->in2out = l_key;
1374       s->out2in = e_key;
1375       s->out2in.protocol = l_key.protocol;
1376       u->nstaticsessions++;
1377
1378       /* Add to lookup tables */
1379       s_kv.value = s - tsm->sessions;
1380       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1381         clib_warning ("in2out-ed key add failed");
1382
1383       key.l_addr = e_key.addr;
1384       key.fib_index = e_key.fib_index;
1385       key.l_port = e_key.port;
1386       s_kv.key[0] = key.as_u64[0];
1387       s_kv.key[1] = key.as_u64[1];
1388       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1389         clib_warning ("out2in-ed key add failed");
1390     }
1391
1392   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1393
1394   /* Update IP checksum */
1395   sum = ip->checksum;
1396   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1397   if (is_twice_nat_session (s))
1398     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1399                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1400   ip->checksum = ip_csum_fold (sum);
1401
1402   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1403     {
1404       old_port = tcp->src_port;
1405       tcp->src_port = s->out2in.port;
1406       new_port = tcp->src_port;
1407
1408       sum = tcp->checksum;
1409       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1410       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1411       if (is_twice_nat_session (s))
1412         {
1413           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1414                                 s->ext_host_addr.as_u32, ip4_header_t,
1415                                 dst_address);
1416           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1417                                 ip4_header_t, length);
1418           tcp->dst_port = s->ext_host_port;
1419           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1420         }
1421       tcp->checksum = ip_csum_fold(sum);
1422     }
1423   else
1424     {
1425       udp->src_port = s->out2in.port;
1426       if (is_twice_nat_session (s))
1427         {
1428           udp->dst_port = s->ext_host_port;
1429           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1430         }
1431       udp->checksum = 0;
1432     }
1433
1434   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1435     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1436
1437   /* Accounting */
1438   s->last_heard = now;
1439   s->total_pkts++;
1440   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1441   /* Per-user LRU list maintenance */
1442   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1443   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1444                       s->per_user_index);
1445   return s;
1446 }
1447
1448 static inline uword
1449 snat_in2out_node_fn_inline (vlib_main_t * vm,
1450                             vlib_node_runtime_t * node,
1451                             vlib_frame_t * frame, int is_slow_path,
1452                             int is_output_feature)
1453 {
1454   u32 n_left_from, * from, * to_next;
1455   snat_in2out_next_t next_index;
1456   u32 pkts_processed = 0;
1457   snat_main_t * sm = &snat_main;
1458   f64 now = vlib_time_now (vm);
1459   u32 stats_node_index;
1460   u32 thread_index = vlib_get_thread_index ();
1461
1462   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1463     snat_in2out_node.index;
1464
1465   from = vlib_frame_vector_args (frame);
1466   n_left_from = frame->n_vectors;
1467   next_index = node->cached_next_index;
1468
1469   while (n_left_from > 0)
1470     {
1471       u32 n_left_to_next;
1472
1473       vlib_get_next_frame (vm, node, next_index,
1474                            to_next, n_left_to_next);
1475
1476       while (n_left_from >= 4 && n_left_to_next >= 2)
1477         {
1478           u32 bi0, bi1;
1479           vlib_buffer_t * b0, * b1;
1480           u32 next0, next1;
1481           u32 sw_if_index0, sw_if_index1;
1482           ip4_header_t * ip0, * ip1;
1483           ip_csum_t sum0, sum1;
1484           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1485           u16 old_port0, new_port0, old_port1, new_port1;
1486           udp_header_t * udp0, * udp1;
1487           tcp_header_t * tcp0, * tcp1;
1488           icmp46_header_t * icmp0, * icmp1;
1489           snat_session_key_t key0, key1;
1490           u32 rx_fib_index0, rx_fib_index1;
1491           u32 proto0, proto1;
1492           snat_session_t * s0 = 0, * s1 = 0;
1493           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1494           u32 iph_offset0 = 0, iph_offset1 = 0;
1495
1496           /* Prefetch next iteration. */
1497           {
1498             vlib_buffer_t * p2, * p3;
1499
1500             p2 = vlib_get_buffer (vm, from[2]);
1501             p3 = vlib_get_buffer (vm, from[3]);
1502
1503             vlib_prefetch_buffer_header (p2, LOAD);
1504             vlib_prefetch_buffer_header (p3, LOAD);
1505
1506             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1507             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1508           }
1509
1510           /* speculatively enqueue b0 and b1 to the current next frame */
1511           to_next[0] = bi0 = from[0];
1512           to_next[1] = bi1 = from[1];
1513           from += 2;
1514           to_next += 2;
1515           n_left_from -= 2;
1516           n_left_to_next -= 2;
1517
1518           b0 = vlib_get_buffer (vm, bi0);
1519           b1 = vlib_get_buffer (vm, bi1);
1520
1521           if (is_output_feature)
1522             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1523
1524           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1525                  iph_offset0);
1526
1527           udp0 = ip4_next_header (ip0);
1528           tcp0 = (tcp_header_t *) udp0;
1529           icmp0 = (icmp46_header_t *) udp0;
1530
1531           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1532           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1533                                    sw_if_index0);
1534
1535           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1536
1537           if (PREDICT_FALSE(ip0->ttl == 1))
1538             {
1539               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1540               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1541                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1542                                            0);
1543               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1544               goto trace00;
1545             }
1546
1547           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1548
1549           /* Next configured feature, probably ip4-lookup */
1550           if (is_slow_path)
1551             {
1552               if (PREDICT_FALSE (proto0 == ~0))
1553                 {
1554                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1555                                                   thread_index, now, vm, node);
1556                   if (!s0)
1557                     next0 = SNAT_IN2OUT_NEXT_DROP;
1558                   goto trace00;
1559                 }
1560
1561               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1562                 {
1563                   next0 = icmp_in2out_slow_path
1564                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1565                      node, next0, now, thread_index, &s0);
1566                   goto trace00;
1567                 }
1568             }
1569           else
1570             {
1571               if (is_output_feature)
1572                 {
1573                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0)))
1574                     goto trace00;
1575                 }
1576
1577               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1578                 {
1579                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1580                   goto trace00;
1581                 }
1582
1583               if (ip4_is_fragment (ip0))
1584                 {
1585                   next0 = SNAT_IN2OUT_NEXT_REASS;
1586                   goto trace00;
1587                 }
1588             }
1589
1590           key0.addr = ip0->src_address;
1591           key0.port = udp0->src_port;
1592           key0.protocol = proto0;
1593           key0.fib_index = rx_fib_index0;
1594
1595           kv0.key = key0.as_u64;
1596
1597           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1598               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1599             {
1600               if (is_slow_path)
1601                 {
1602                   if (is_output_feature)
1603                     {
1604                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1605                           ip0, proto0, thread_index)))
1606                         goto trace00;
1607                     }
1608                   else
1609                     {
1610                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1611                           ip0, proto0, rx_fib_index0, thread_index)))
1612                         goto trace00;
1613                     }
1614
1615                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1616                                      &s0, node, next0, thread_index);
1617                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1618                     goto trace00;
1619                 }
1620               else
1621                 {
1622                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1623                   goto trace00;
1624                 }
1625             }
1626           else
1627             {
1628               if (PREDICT_FALSE (value0.value == ~0ULL))
1629                 {
1630                   if (is_slow_path)
1631                     {
1632                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1633                                           thread_index, now, vm, node);
1634                       if (!s0 && !sm->forwarding_enabled)
1635                         next0 = SNAT_IN2OUT_NEXT_DROP;
1636                       goto trace00;
1637                     }
1638                   else
1639                     {
1640                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1641                       goto trace00;
1642                     }
1643                 }
1644               else
1645                 {
1646                   s0 = pool_elt_at_index (
1647                     sm->per_thread_data[thread_index].sessions,
1648                     value0.value);
1649                 }
1650             }
1651
1652           b0->flags |= VNET_BUFFER_F_IS_NATED;
1653
1654           old_addr0 = ip0->src_address.as_u32;
1655           ip0->src_address = s0->out2in.addr;
1656           new_addr0 = ip0->src_address.as_u32;
1657           if (!is_output_feature)
1658             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1659
1660           sum0 = ip0->checksum;
1661           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1662                                  ip4_header_t,
1663                                  src_address /* changed member */);
1664           ip0->checksum = ip_csum_fold (sum0);
1665
1666           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1667             {
1668               old_port0 = tcp0->src_port;
1669               tcp0->src_port = s0->out2in.port;
1670               new_port0 = tcp0->src_port;
1671
1672               sum0 = tcp0->checksum;
1673               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1674                                      ip4_header_t,
1675                                      dst_address /* changed member */);
1676               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1677                                      ip4_header_t /* cheat */,
1678                                      length /* changed member */);
1679               tcp0->checksum = ip_csum_fold(sum0);
1680             }
1681           else
1682             {
1683               old_port0 = udp0->src_port;
1684               udp0->src_port = s0->out2in.port;
1685               udp0->checksum = 0;
1686             }
1687
1688           /* Accounting */
1689           s0->last_heard = now;
1690           s0->total_pkts++;
1691           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1692           /* Per-user LRU list maintenance */
1693           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1694                              s0->per_user_index);
1695           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1696                               s0->per_user_list_head_index,
1697                               s0->per_user_index);
1698         trace00:
1699
1700           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1701                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1702             {
1703               snat_in2out_trace_t *t =
1704                  vlib_add_trace (vm, node, b0, sizeof (*t));
1705               t->is_slow_path = is_slow_path;
1706               t->sw_if_index = sw_if_index0;
1707               t->next_index = next0;
1708                   t->session_index = ~0;
1709               if (s0)
1710                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1711             }
1712
1713           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1714
1715           if (is_output_feature)
1716             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1717
1718           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1719                  iph_offset1);
1720
1721           udp1 = ip4_next_header (ip1);
1722           tcp1 = (tcp_header_t *) udp1;
1723           icmp1 = (icmp46_header_t *) udp1;
1724
1725           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1726           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1727                                    sw_if_index1);
1728
1729           if (PREDICT_FALSE(ip1->ttl == 1))
1730             {
1731               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1732               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1733                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1734                                            0);
1735               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1736               goto trace01;
1737             }
1738
1739           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1740
1741           /* Next configured feature, probably ip4-lookup */
1742           if (is_slow_path)
1743             {
1744               if (PREDICT_FALSE (proto1 == ~0))
1745                 {
1746                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1747                                                   thread_index, now, vm, node);
1748                   if (!s1)
1749                     next1 = SNAT_IN2OUT_NEXT_DROP;
1750                   goto trace01;
1751                 }
1752
1753               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1754                 {
1755                   next1 = icmp_in2out_slow_path
1756                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1757                      next1, now, thread_index, &s1);
1758                   goto trace01;
1759                 }
1760             }
1761           else
1762             {
1763               if (is_output_feature)
1764                 {
1765                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip1)))
1766                     goto trace01;
1767                 }
1768
1769               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1770                 {
1771                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1772                   goto trace01;
1773                 }
1774
1775               if (ip4_is_fragment (ip1))
1776                 {
1777                   next1 = SNAT_IN2OUT_NEXT_REASS;
1778                   goto trace01;
1779                 }
1780             }
1781
1782           key1.addr = ip1->src_address;
1783           key1.port = udp1->src_port;
1784           key1.protocol = proto1;
1785           key1.fib_index = rx_fib_index1;
1786
1787           kv1.key = key1.as_u64;
1788
1789             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1790                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1791             {
1792               if (is_slow_path)
1793                 {
1794                   if (is_output_feature)
1795                     {
1796                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1797                           ip1, proto1, thread_index)))
1798                         goto trace00;
1799                     }
1800                   else
1801                     {
1802                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1803                           ip1, proto1, rx_fib_index1, thread_index)))
1804                         goto trace01;
1805                     }
1806
1807                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1808                                      &s1, node, next1, thread_index);
1809                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1810                     goto trace01;
1811                 }
1812               else
1813                 {
1814                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1815                   goto trace01;
1816                 }
1817             }
1818           else
1819             {
1820               if (PREDICT_FALSE (value1.value == ~0ULL))
1821                 {
1822                   if (is_slow_path)
1823                     {
1824                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1825                                           thread_index, now, vm, node);
1826                       if (!s1 && !sm->forwarding_enabled)
1827                         next1 = SNAT_IN2OUT_NEXT_DROP;
1828                       goto trace01;
1829                     }
1830                   else
1831                     {
1832                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1833                       goto trace01;
1834                     }
1835                 }
1836               else
1837                 {
1838                   s1 = pool_elt_at_index (
1839                     sm->per_thread_data[thread_index].sessions,
1840                     value1.value);
1841                 }
1842             }
1843
1844           b1->flags |= VNET_BUFFER_F_IS_NATED;
1845
1846           old_addr1 = ip1->src_address.as_u32;
1847           ip1->src_address = s1->out2in.addr;
1848           new_addr1 = ip1->src_address.as_u32;
1849           if (!is_output_feature)
1850             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1851
1852           sum1 = ip1->checksum;
1853           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1854                                  ip4_header_t,
1855                                  src_address /* changed member */);
1856           ip1->checksum = ip_csum_fold (sum1);
1857
1858           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1859             {
1860               old_port1 = tcp1->src_port;
1861               tcp1->src_port = s1->out2in.port;
1862               new_port1 = tcp1->src_port;
1863
1864               sum1 = tcp1->checksum;
1865               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1866                                      ip4_header_t,
1867                                      dst_address /* changed member */);
1868               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1869                                      ip4_header_t /* cheat */,
1870                                      length /* changed member */);
1871               tcp1->checksum = ip_csum_fold(sum1);
1872             }
1873           else
1874             {
1875               old_port1 = udp1->src_port;
1876               udp1->src_port = s1->out2in.port;
1877               udp1->checksum = 0;
1878             }
1879
1880           /* Accounting */
1881           s1->last_heard = now;
1882           s1->total_pkts++;
1883           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1884           /* Per-user LRU list maintenance */
1885           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1886                              s1->per_user_index);
1887           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1888                               s1->per_user_list_head_index,
1889                               s1->per_user_index);
1890         trace01:
1891
1892           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1893                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1894             {
1895               snat_in2out_trace_t *t =
1896                  vlib_add_trace (vm, node, b1, sizeof (*t));
1897               t->sw_if_index = sw_if_index1;
1898               t->next_index = next1;
1899               t->session_index = ~0;
1900               if (s1)
1901                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1902             }
1903
1904           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1905
1906           /* verify speculative enqueues, maybe switch current next frame */
1907           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1908                                            to_next, n_left_to_next,
1909                                            bi0, bi1, next0, next1);
1910         }
1911
1912       while (n_left_from > 0 && n_left_to_next > 0)
1913         {
1914           u32 bi0;
1915           vlib_buffer_t * b0;
1916           u32 next0;
1917           u32 sw_if_index0;
1918           ip4_header_t * ip0;
1919           ip_csum_t sum0;
1920           u32 new_addr0, old_addr0;
1921           u16 old_port0, new_port0;
1922           udp_header_t * udp0;
1923           tcp_header_t * tcp0;
1924           icmp46_header_t * icmp0;
1925           snat_session_key_t key0;
1926           u32 rx_fib_index0;
1927           u32 proto0;
1928           snat_session_t * s0 = 0;
1929           clib_bihash_kv_8_8_t kv0, value0;
1930           u32 iph_offset0 = 0;
1931
1932           /* speculatively enqueue b0 to the current next frame */
1933           bi0 = from[0];
1934           to_next[0] = bi0;
1935           from += 1;
1936           to_next += 1;
1937           n_left_from -= 1;
1938           n_left_to_next -= 1;
1939
1940           b0 = vlib_get_buffer (vm, bi0);
1941           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1942
1943           if (is_output_feature)
1944             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1945
1946           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1947                  iph_offset0);
1948
1949           udp0 = ip4_next_header (ip0);
1950           tcp0 = (tcp_header_t *) udp0;
1951           icmp0 = (icmp46_header_t *) udp0;
1952
1953           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1954           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1955                                    sw_if_index0);
1956
1957           if (PREDICT_FALSE(ip0->ttl == 1))
1958             {
1959               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1960               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1961                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1962                                            0);
1963               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1964               goto trace0;
1965             }
1966
1967           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1968
1969           /* Next configured feature, probably ip4-lookup */
1970           if (is_slow_path)
1971             {
1972               if (PREDICT_FALSE (proto0 == ~0))
1973                 {
1974                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1975                                                   thread_index, now, vm, node);
1976                   if (!s0)
1977                     next0 = SNAT_IN2OUT_NEXT_DROP;
1978                   goto trace0;
1979                 }
1980
1981               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1982                 {
1983                   next0 = icmp_in2out_slow_path
1984                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1985                      next0, now, thread_index, &s0);
1986                   goto trace0;
1987                 }
1988             }
1989           else
1990             {
1991                if (is_output_feature)
1992                 {
1993                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0)))
1994                     goto trace0;
1995                 }
1996
1997               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1998                 {
1999                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2000                   goto trace0;
2001                 }
2002
2003               if (ip4_is_fragment (ip0))
2004                 {
2005                   next0 = SNAT_IN2OUT_NEXT_REASS;
2006                   goto trace0;
2007                 }
2008             }
2009
2010           key0.addr = ip0->src_address;
2011           key0.port = udp0->src_port;
2012           key0.protocol = proto0;
2013           key0.fib_index = rx_fib_index0;
2014
2015           kv0.key = key0.as_u64;
2016
2017           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2018                                       &kv0, &value0))
2019             {
2020               if (is_slow_path)
2021                 {
2022                   if (is_output_feature)
2023                     {
2024                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
2025                           ip0, proto0, thread_index)))
2026                         goto trace0;
2027                     }
2028                   else
2029                     {
2030                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2031                           ip0, proto0, rx_fib_index0, thread_index)))
2032                         goto trace0;
2033                     }
2034
2035                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2036                                      &s0, node, next0, thread_index);
2037
2038                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2039                     goto trace0;
2040                 }
2041               else
2042                 {
2043                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2044                   goto trace0;
2045                 }
2046             }
2047           else
2048             {
2049               if (PREDICT_FALSE (value0.value == ~0ULL))
2050                 {
2051                   if (is_slow_path)
2052                     {
2053                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2054                                           thread_index, now, vm, node);
2055                       if (!s0 && !sm->forwarding_enabled)
2056                         next0 = SNAT_IN2OUT_NEXT_DROP;
2057                       goto trace0;
2058                     }
2059                   else
2060                     {
2061                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2062                       goto trace0;
2063                     }
2064                 }
2065               else
2066                 {
2067                   s0 = pool_elt_at_index (
2068                     sm->per_thread_data[thread_index].sessions,
2069                     value0.value);
2070                 }
2071             }
2072
2073           b0->flags |= VNET_BUFFER_F_IS_NATED;
2074
2075           old_addr0 = ip0->src_address.as_u32;
2076           ip0->src_address = s0->out2in.addr;
2077           new_addr0 = ip0->src_address.as_u32;
2078           if (!is_output_feature)
2079             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2080
2081           sum0 = ip0->checksum;
2082           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2083                                  ip4_header_t,
2084                                  src_address /* changed member */);
2085           ip0->checksum = ip_csum_fold (sum0);
2086
2087           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2088             {
2089               old_port0 = tcp0->src_port;
2090               tcp0->src_port = s0->out2in.port;
2091               new_port0 = tcp0->src_port;
2092
2093               sum0 = tcp0->checksum;
2094               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2095                                      ip4_header_t,
2096                                      dst_address /* changed member */);
2097               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2098                                      ip4_header_t /* cheat */,
2099                                      length /* changed member */);
2100               tcp0->checksum = ip_csum_fold(sum0);
2101             }
2102           else
2103             {
2104               old_port0 = udp0->src_port;
2105               udp0->src_port = s0->out2in.port;
2106               udp0->checksum = 0;
2107             }
2108
2109           /* Accounting */
2110           s0->last_heard = now;
2111           s0->total_pkts++;
2112           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2113           /* Per-user LRU list maintenance */
2114           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2115                              s0->per_user_index);
2116           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2117                               s0->per_user_list_head_index,
2118                               s0->per_user_index);
2119
2120         trace0:
2121           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2122                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2123             {
2124               snat_in2out_trace_t *t =
2125                  vlib_add_trace (vm, node, b0, sizeof (*t));
2126               t->is_slow_path = is_slow_path;
2127               t->sw_if_index = sw_if_index0;
2128               t->next_index = next0;
2129                   t->session_index = ~0;
2130               if (s0)
2131                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2132             }
2133
2134           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2135
2136           /* verify speculative enqueue, maybe switch current next frame */
2137           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2138                                            to_next, n_left_to_next,
2139                                            bi0, next0);
2140         }
2141
2142       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2143     }
2144
2145   vlib_node_increment_counter (vm, stats_node_index,
2146                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2147                                pkts_processed);
2148   return frame->n_vectors;
2149 }
2150
2151 static uword
2152 snat_in2out_fast_path_fn (vlib_main_t * vm,
2153                           vlib_node_runtime_t * node,
2154                           vlib_frame_t * frame)
2155 {
2156   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2157 }
2158
2159 VLIB_REGISTER_NODE (snat_in2out_node) = {
2160   .function = snat_in2out_fast_path_fn,
2161   .name = "nat44-in2out",
2162   .vector_size = sizeof (u32),
2163   .format_trace = format_snat_in2out_trace,
2164   .type = VLIB_NODE_TYPE_INTERNAL,
2165
2166   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2167   .error_strings = snat_in2out_error_strings,
2168
2169   .runtime_data_bytes = sizeof (snat_runtime_t),
2170
2171   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2172
2173   /* edit / add dispositions here */
2174   .next_nodes = {
2175     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2176     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2177     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2178     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2179     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2180   },
2181 };
2182
2183 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2184
2185 static uword
2186 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2187                                  vlib_node_runtime_t * node,
2188                                  vlib_frame_t * frame)
2189 {
2190   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2191 }
2192
2193 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2194   .function = snat_in2out_output_fast_path_fn,
2195   .name = "nat44-in2out-output",
2196   .vector_size = sizeof (u32),
2197   .format_trace = format_snat_in2out_trace,
2198   .type = VLIB_NODE_TYPE_INTERNAL,
2199
2200   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2201   .error_strings = snat_in2out_error_strings,
2202
2203   .runtime_data_bytes = sizeof (snat_runtime_t),
2204
2205   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2206
2207   /* edit / add dispositions here */
2208   .next_nodes = {
2209     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2210     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2211     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2212     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2213     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2214   },
2215 };
2216
2217 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2218                               snat_in2out_output_fast_path_fn);
2219
2220 static uword
2221 snat_in2out_slow_path_fn (vlib_main_t * vm,
2222                           vlib_node_runtime_t * node,
2223                           vlib_frame_t * frame)
2224 {
2225   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2226 }
2227
2228 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2229   .function = snat_in2out_slow_path_fn,
2230   .name = "nat44-in2out-slowpath",
2231   .vector_size = sizeof (u32),
2232   .format_trace = format_snat_in2out_trace,
2233   .type = VLIB_NODE_TYPE_INTERNAL,
2234
2235   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2236   .error_strings = snat_in2out_error_strings,
2237
2238   .runtime_data_bytes = sizeof (snat_runtime_t),
2239
2240   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2241
2242   /* edit / add dispositions here */
2243   .next_nodes = {
2244     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2245     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2246     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2247     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2248     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2249   },
2250 };
2251
2252 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2253                               snat_in2out_slow_path_fn);
2254
2255 static uword
2256 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2257                                  vlib_node_runtime_t * node,
2258                                  vlib_frame_t * frame)
2259 {
2260   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2261 }
2262
2263 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2264   .function = snat_in2out_output_slow_path_fn,
2265   .name = "nat44-in2out-output-slowpath",
2266   .vector_size = sizeof (u32),
2267   .format_trace = format_snat_in2out_trace,
2268   .type = VLIB_NODE_TYPE_INTERNAL,
2269
2270   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2271   .error_strings = snat_in2out_error_strings,
2272
2273   .runtime_data_bytes = sizeof (snat_runtime_t),
2274
2275   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2276
2277   /* edit / add dispositions here */
2278   .next_nodes = {
2279     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2280     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2281     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2282     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2283     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2284   },
2285 };
2286
2287 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2288                               snat_in2out_output_slow_path_fn);
2289
2290 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2291
2292 static uword
2293 nat44_hairpinning_fn (vlib_main_t * vm,
2294                       vlib_node_runtime_t * node,
2295                       vlib_frame_t * frame)
2296 {
2297   u32 n_left_from, * from, * to_next;
2298   snat_in2out_next_t next_index;
2299   u32 pkts_processed = 0;
2300   snat_main_t * sm = &snat_main;
2301   vnet_feature_main_t *fm = &feature_main;
2302   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2303   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2304
2305   from = vlib_frame_vector_args (frame);
2306   n_left_from = frame->n_vectors;
2307   next_index = node->cached_next_index;
2308
2309   while (n_left_from > 0)
2310     {
2311       u32 n_left_to_next;
2312
2313       vlib_get_next_frame (vm, node, next_index,
2314                            to_next, n_left_to_next);
2315
2316       while (n_left_from > 0 && n_left_to_next > 0)
2317         {
2318           u32 bi0;
2319           vlib_buffer_t * b0;
2320           u32 next0;
2321           ip4_header_t * ip0;
2322           u32 proto0;
2323           udp_header_t * udp0;
2324           tcp_header_t * tcp0;
2325
2326           /* speculatively enqueue b0 to the current next frame */
2327           bi0 = from[0];
2328           to_next[0] = bi0;
2329           from += 1;
2330           to_next += 1;
2331           n_left_from -= 1;
2332           n_left_to_next -= 1;
2333
2334           b0 = vlib_get_buffer (vm, bi0);
2335           ip0 = vlib_buffer_get_current (b0);
2336           udp0 = ip4_next_header (ip0);
2337           tcp0 = (tcp_header_t *) udp0;
2338
2339           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2340
2341           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2342                                 &next0, 0);
2343
2344           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2345             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2346
2347           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2348
2349           /* verify speculative enqueue, maybe switch current next frame */
2350           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2351                                            to_next, n_left_to_next,
2352                                            bi0, next0);
2353          }
2354
2355       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2356     }
2357
2358   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2359                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2360                                pkts_processed);
2361   return frame->n_vectors;
2362 }
2363
2364 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2365   .function = nat44_hairpinning_fn,
2366   .name = "nat44-hairpinning",
2367   .vector_size = sizeof (u32),
2368   .type = VLIB_NODE_TYPE_INTERNAL,
2369   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2370   .error_strings = snat_in2out_error_strings,
2371   .n_next_nodes = 2,
2372   .next_nodes = {
2373     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2374     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2375   },
2376 };
2377
2378 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2379                               nat44_hairpinning_fn);
2380
2381 static inline void
2382 nat44_reass_hairpinning (snat_main_t *sm,
2383                          vlib_buffer_t * b0,
2384                          ip4_header_t * ip0,
2385                          u16 sport,
2386                          u16 dport,
2387                          u32 proto0)
2388 {
2389   snat_session_key_t key0, sm0;
2390   snat_session_t * s0;
2391   clib_bihash_kv_8_8_t kv0, value0;
2392   ip_csum_t sum0;
2393   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2394   u16 new_dst_port0, old_dst_port0;
2395   udp_header_t * udp0;
2396   tcp_header_t * tcp0;
2397
2398   key0.addr = ip0->dst_address;
2399   key0.port = dport;
2400   key0.protocol = proto0;
2401   key0.fib_index = sm->outside_fib_index;
2402   kv0.key = key0.as_u64;
2403
2404   udp0 = ip4_next_header (ip0);
2405
2406   /* Check if destination is static mappings */
2407   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0))
2408     {
2409       new_dst_addr0 = sm0.addr.as_u32;
2410       new_dst_port0 = sm0.port;
2411       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2412     }
2413   /* or active sessions */
2414   else
2415     {
2416       if (sm->num_workers > 1)
2417         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2418       else
2419         ti = sm->num_workers;
2420
2421       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2422         {
2423           si = value0.value;
2424           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2425           new_dst_addr0 = s0->in2out.addr.as_u32;
2426           new_dst_port0 = s0->in2out.port;
2427           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2428         }
2429     }
2430
2431   /* Destination is behind the same NAT, use internal address and port */
2432   if (new_dst_addr0)
2433     {
2434       old_dst_addr0 = ip0->dst_address.as_u32;
2435       ip0->dst_address.as_u32 = new_dst_addr0;
2436       sum0 = ip0->checksum;
2437       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2438                              ip4_header_t, dst_address);
2439       ip0->checksum = ip_csum_fold (sum0);
2440
2441       old_dst_port0 = dport;
2442       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2443                        ip4_is_first_fragment (ip0)))
2444         {
2445           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2446             {
2447               tcp0 = ip4_next_header (ip0);
2448               tcp0->dst = new_dst_port0;
2449               sum0 = tcp0->checksum;
2450               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2451                                      ip4_header_t, dst_address);
2452               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2453                                      ip4_header_t /* cheat */, length);
2454               tcp0->checksum = ip_csum_fold(sum0);
2455             }
2456           else
2457             {
2458               udp0->dst_port = new_dst_port0;
2459               udp0->checksum = 0;
2460             }
2461         }
2462       else
2463         {
2464           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2465             {
2466               tcp0 = ip4_next_header (ip0);
2467               sum0 = tcp0->checksum;
2468               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2469                                      ip4_header_t, dst_address);
2470               tcp0->checksum = ip_csum_fold(sum0);
2471             }
2472         }
2473     }
2474 }
2475
2476 static uword
2477 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2478                             vlib_node_runtime_t * node,
2479                             vlib_frame_t * frame)
2480 {
2481   u32 n_left_from, *from, *to_next;
2482   snat_in2out_next_t next_index;
2483   u32 pkts_processed = 0;
2484   snat_main_t *sm = &snat_main;
2485   f64 now = vlib_time_now (vm);
2486   u32 thread_index = vlib_get_thread_index ();
2487   snat_main_per_thread_data_t *per_thread_data =
2488     &sm->per_thread_data[thread_index];
2489   u32 *fragments_to_drop = 0;
2490   u32 *fragments_to_loopback = 0;
2491
2492   from = vlib_frame_vector_args (frame);
2493   n_left_from = frame->n_vectors;
2494   next_index = node->cached_next_index;
2495
2496   while (n_left_from > 0)
2497     {
2498       u32 n_left_to_next;
2499
2500       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2501
2502       while (n_left_from > 0 && n_left_to_next > 0)
2503        {
2504           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2505           vlib_buffer_t *b0;
2506           u32 next0;
2507           u8 cached0 = 0;
2508           ip4_header_t *ip0;
2509           nat_reass_ip4_t *reass0;
2510           udp_header_t * udp0;
2511           tcp_header_t * tcp0;
2512           snat_session_key_t key0;
2513           clib_bihash_kv_8_8_t kv0, value0;
2514           snat_session_t * s0 = 0;
2515           u16 old_port0, new_port0;
2516           ip_csum_t sum0;
2517
2518           /* speculatively enqueue b0 to the current next frame */
2519           bi0 = from[0];
2520           to_next[0] = bi0;
2521           from += 1;
2522           to_next += 1;
2523           n_left_from -= 1;
2524           n_left_to_next -= 1;
2525
2526           b0 = vlib_get_buffer (vm, bi0);
2527           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2528
2529           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2530           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2531                                                                sw_if_index0);
2532
2533           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2534             {
2535               next0 = SNAT_IN2OUT_NEXT_DROP;
2536               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2537               goto trace0;
2538             }
2539
2540           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2541           udp0 = ip4_next_header (ip0);
2542           tcp0 = (tcp_header_t *) udp0;
2543           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2544
2545           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2546                                                  ip0->dst_address,
2547                                                  ip0->fragment_id,
2548                                                  ip0->protocol,
2549                                                  1,
2550                                                  &fragments_to_drop);
2551
2552           if (PREDICT_FALSE (!reass0))
2553             {
2554               next0 = SNAT_IN2OUT_NEXT_DROP;
2555               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2556               goto trace0;
2557             }
2558
2559           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2560             {
2561               key0.addr = ip0->src_address;
2562               key0.port = udp0->src_port;
2563               key0.protocol = proto0;
2564               key0.fib_index = rx_fib_index0;
2565               kv0.key = key0.as_u64;
2566
2567               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2568                 {
2569                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2570                       ip0, proto0, rx_fib_index0, thread_index)))
2571                     goto trace0;
2572
2573                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2574                                      &s0, node, next0, thread_index);
2575
2576                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2577                     goto trace0;
2578
2579                   reass0->sess_index = s0 - per_thread_data->sessions;
2580                 }
2581               else
2582                 {
2583                   s0 = pool_elt_at_index (per_thread_data->sessions,
2584                                           value0.value);
2585                   reass0->sess_index = value0.value;
2586                 }
2587               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2588             }
2589           else
2590             {
2591               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2592                 {
2593                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2594                     {
2595                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2596                       next0 = SNAT_IN2OUT_NEXT_DROP;
2597                       goto trace0;
2598                     }
2599                   cached0 = 1;
2600                   goto trace0;
2601                 }
2602               s0 = pool_elt_at_index (per_thread_data->sessions,
2603                                       reass0->sess_index);
2604             }
2605
2606           old_addr0 = ip0->src_address.as_u32;
2607           ip0->src_address = s0->out2in.addr;
2608           new_addr0 = ip0->src_address.as_u32;
2609           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2610
2611           sum0 = ip0->checksum;
2612           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2613                                  ip4_header_t,
2614                                  src_address /* changed member */);
2615           ip0->checksum = ip_csum_fold (sum0);
2616
2617           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2618             {
2619               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2620                 {
2621                   old_port0 = tcp0->src_port;
2622                   tcp0->src_port = s0->out2in.port;
2623                   new_port0 = tcp0->src_port;
2624
2625                   sum0 = tcp0->checksum;
2626                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2627                                          ip4_header_t,
2628                                          dst_address /* changed member */);
2629                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2630                                          ip4_header_t /* cheat */,
2631                                          length /* changed member */);
2632                   tcp0->checksum = ip_csum_fold(sum0);
2633                 }
2634               else
2635                 {
2636                   old_port0 = udp0->src_port;
2637                   udp0->src_port = s0->out2in.port;
2638                   udp0->checksum = 0;
2639                 }
2640             }
2641
2642           /* Hairpinning */
2643           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2644                                    s0->ext_host_port, proto0);
2645
2646           /* Accounting */
2647           s0->last_heard = now;
2648           s0->total_pkts++;
2649           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2650           /* Per-user LRU list maintenance */
2651           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2652                              s0->per_user_index);
2653           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2654                               s0->per_user_list_head_index,
2655                               s0->per_user_index);
2656
2657         trace0:
2658           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2659                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2660             {
2661               nat44_in2out_reass_trace_t *t =
2662                  vlib_add_trace (vm, node, b0, sizeof (*t));
2663               t->cached = cached0;
2664               t->sw_if_index = sw_if_index0;
2665               t->next_index = next0;
2666             }
2667
2668           if (cached0)
2669             {
2670               n_left_to_next++;
2671               to_next--;
2672             }
2673           else
2674             {
2675               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2676
2677               /* verify speculative enqueue, maybe switch current next frame */
2678               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2679                                                to_next, n_left_to_next,
2680                                                bi0, next0);
2681             }
2682
2683           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2684             {
2685               from = vlib_frame_vector_args (frame);
2686               u32 len = vec_len (fragments_to_loopback);
2687               if (len <= VLIB_FRAME_SIZE)
2688                 {
2689                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2690                   n_left_from = len;
2691                   vec_reset_length (fragments_to_loopback);
2692                 }
2693               else
2694                 {
2695                   clib_memcpy (from,
2696                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2697                                sizeof (u32) * VLIB_FRAME_SIZE);
2698                   n_left_from = VLIB_FRAME_SIZE;
2699                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2700                 }
2701             }
2702        }
2703
2704       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2705     }
2706
2707   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2708                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2709                                pkts_processed);
2710
2711   nat_send_all_to_node (vm, fragments_to_drop, node,
2712                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2713                         SNAT_IN2OUT_NEXT_DROP);
2714
2715   vec_free (fragments_to_drop);
2716   vec_free (fragments_to_loopback);
2717   return frame->n_vectors;
2718 }
2719
2720 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2721   .function = nat44_in2out_reass_node_fn,
2722   .name = "nat44-in2out-reass",
2723   .vector_size = sizeof (u32),
2724   .format_trace = format_nat44_in2out_reass_trace,
2725   .type = VLIB_NODE_TYPE_INTERNAL,
2726
2727   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2728   .error_strings = snat_in2out_error_strings,
2729
2730   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2731   .next_nodes = {
2732     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2733     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2734     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2735     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2736     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2737   },
2738 };
2739
2740 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2741                               nat44_in2out_reass_node_fn);
2742
2743 /**************************/
2744 /*** deterministic mode ***/
2745 /**************************/
2746 static uword
2747 snat_det_in2out_node_fn (vlib_main_t * vm,
2748                          vlib_node_runtime_t * node,
2749                          vlib_frame_t * frame)
2750 {
2751   u32 n_left_from, * from, * to_next;
2752   snat_in2out_next_t next_index;
2753   u32 pkts_processed = 0;
2754   snat_main_t * sm = &snat_main;
2755   u32 now = (u32) vlib_time_now (vm);
2756   u32 thread_index = vlib_get_thread_index ();
2757
2758   from = vlib_frame_vector_args (frame);
2759   n_left_from = frame->n_vectors;
2760   next_index = node->cached_next_index;
2761
2762   while (n_left_from > 0)
2763     {
2764       u32 n_left_to_next;
2765
2766       vlib_get_next_frame (vm, node, next_index,
2767                            to_next, n_left_to_next);
2768
2769       while (n_left_from >= 4 && n_left_to_next >= 2)
2770         {
2771           u32 bi0, bi1;
2772           vlib_buffer_t * b0, * b1;
2773           u32 next0, next1;
2774           u32 sw_if_index0, sw_if_index1;
2775           ip4_header_t * ip0, * ip1;
2776           ip_csum_t sum0, sum1;
2777           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2778           u16 old_port0, new_port0, lo_port0, i0;
2779           u16 old_port1, new_port1, lo_port1, i1;
2780           udp_header_t * udp0, * udp1;
2781           tcp_header_t * tcp0, * tcp1;
2782           u32 proto0, proto1;
2783           snat_det_out_key_t key0, key1;
2784           snat_det_map_t * dm0, * dm1;
2785           snat_det_session_t * ses0 = 0, * ses1 = 0;
2786           u32 rx_fib_index0, rx_fib_index1;
2787           icmp46_header_t * icmp0, * icmp1;
2788
2789           /* Prefetch next iteration. */
2790           {
2791             vlib_buffer_t * p2, * p3;
2792
2793             p2 = vlib_get_buffer (vm, from[2]);
2794             p3 = vlib_get_buffer (vm, from[3]);
2795
2796             vlib_prefetch_buffer_header (p2, LOAD);
2797             vlib_prefetch_buffer_header (p3, LOAD);
2798
2799             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2800             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2801           }
2802
2803           /* speculatively enqueue b0 and b1 to the current next frame */
2804           to_next[0] = bi0 = from[0];
2805           to_next[1] = bi1 = from[1];
2806           from += 2;
2807           to_next += 2;
2808           n_left_from -= 2;
2809           n_left_to_next -= 2;
2810
2811           b0 = vlib_get_buffer (vm, bi0);
2812           b1 = vlib_get_buffer (vm, bi1);
2813
2814           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2815           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2816
2817           ip0 = vlib_buffer_get_current (b0);
2818           udp0 = ip4_next_header (ip0);
2819           tcp0 = (tcp_header_t *) udp0;
2820
2821           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2822
2823           if (PREDICT_FALSE(ip0->ttl == 1))
2824             {
2825               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2826               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2827                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2828                                            0);
2829               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2830               goto trace0;
2831             }
2832
2833           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2834
2835           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2836             {
2837               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2838               icmp0 = (icmp46_header_t *) udp0;
2839
2840               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2841                                   rx_fib_index0, node, next0, thread_index,
2842                                   &ses0, &dm0);
2843               goto trace0;
2844             }
2845
2846           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2847           if (PREDICT_FALSE(!dm0))
2848             {
2849               clib_warning("no match for internal host %U",
2850                            format_ip4_address, &ip0->src_address);
2851               next0 = SNAT_IN2OUT_NEXT_DROP;
2852               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2853               goto trace0;
2854             }
2855
2856           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2857
2858           key0.ext_host_addr = ip0->dst_address;
2859           key0.ext_host_port = tcp0->dst;
2860
2861           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2862           if (PREDICT_FALSE(!ses0))
2863             {
2864               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2865                 {
2866                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2867                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2868
2869                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2870                     continue;
2871
2872                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2873                   break;
2874                 }
2875               if (PREDICT_FALSE(!ses0))
2876                 {
2877                   /* too many sessions for user, send ICMP error packet */
2878
2879                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2880                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2881                                                ICMP4_destination_unreachable_destination_unreachable_host,
2882                                                0);
2883                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2884                   goto trace0;
2885                 }
2886             }
2887
2888           new_port0 = ses0->out.out_port;
2889
2890           old_addr0.as_u32 = ip0->src_address.as_u32;
2891           ip0->src_address.as_u32 = new_addr0.as_u32;
2892           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2893
2894           sum0 = ip0->checksum;
2895           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2896                                  ip4_header_t,
2897                                  src_address /* changed member */);
2898           ip0->checksum = ip_csum_fold (sum0);
2899
2900           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2901             {
2902               if (tcp0->flags & TCP_FLAG_SYN)
2903                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2904               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2905                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2906               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2907                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2908               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2909                 snat_det_ses_close(dm0, ses0);
2910               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2911                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2912               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2913                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2914
2915               old_port0 = tcp0->src;
2916               tcp0->src = new_port0;
2917
2918               sum0 = tcp0->checksum;
2919               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2920                                      ip4_header_t,
2921                                      dst_address /* changed member */);
2922               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2923                                      ip4_header_t /* cheat */,
2924                                      length /* changed member */);
2925               tcp0->checksum = ip_csum_fold(sum0);
2926             }
2927           else
2928             {
2929               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2930               old_port0 = udp0->src_port;
2931               udp0->src_port = new_port0;
2932               udp0->checksum = 0;
2933             }
2934
2935           switch(ses0->state)
2936             {
2937             case SNAT_SESSION_UDP_ACTIVE:
2938                 ses0->expire = now + sm->udp_timeout;
2939                 break;
2940             case SNAT_SESSION_TCP_SYN_SENT:
2941             case SNAT_SESSION_TCP_FIN_WAIT:
2942             case SNAT_SESSION_TCP_CLOSE_WAIT:
2943             case SNAT_SESSION_TCP_LAST_ACK:
2944                 ses0->expire = now + sm->tcp_transitory_timeout;
2945                 break;
2946             case SNAT_SESSION_TCP_ESTABLISHED:
2947                 ses0->expire = now + sm->tcp_established_timeout;
2948                 break;
2949             }
2950
2951         trace0:
2952           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2953                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2954             {
2955               snat_in2out_trace_t *t =
2956                  vlib_add_trace (vm, node, b0, sizeof (*t));
2957               t->is_slow_path = 0;
2958               t->sw_if_index = sw_if_index0;
2959               t->next_index = next0;
2960               t->session_index = ~0;
2961               if (ses0)
2962                 t->session_index = ses0 - dm0->sessions;
2963             }
2964
2965           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2966
2967           ip1 = vlib_buffer_get_current (b1);
2968           udp1 = ip4_next_header (ip1);
2969           tcp1 = (tcp_header_t *) udp1;
2970
2971           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
2972
2973           if (PREDICT_FALSE(ip1->ttl == 1))
2974             {
2975               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2976               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
2977                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2978                                            0);
2979               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2980               goto trace1;
2981             }
2982
2983           proto1 = ip_proto_to_snat_proto (ip1->protocol);
2984
2985           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
2986             {
2987               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
2988               icmp1 = (icmp46_header_t *) udp1;
2989
2990               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
2991                                   rx_fib_index1, node, next1, thread_index,
2992                                   &ses1, &dm1);
2993               goto trace1;
2994             }
2995
2996           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
2997           if (PREDICT_FALSE(!dm1))
2998             {
2999               clib_warning("no match for internal host %U",
3000                            format_ip4_address, &ip0->src_address);
3001               next1 = SNAT_IN2OUT_NEXT_DROP;
3002               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3003               goto trace1;
3004             }
3005
3006           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3007
3008           key1.ext_host_addr = ip1->dst_address;
3009           key1.ext_host_port = tcp1->dst;
3010
3011           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3012           if (PREDICT_FALSE(!ses1))
3013             {
3014               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3015                 {
3016                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
3017                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3018
3019                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3020                     continue;
3021
3022                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3023                   break;
3024                 }
3025               if (PREDICT_FALSE(!ses1))
3026                 {
3027                   /* too many sessions for user, send ICMP error packet */
3028
3029                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3030                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3031                                                ICMP4_destination_unreachable_destination_unreachable_host,
3032                                                0);
3033                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3034                   goto trace1;
3035                 }
3036             }
3037
3038           new_port1 = ses1->out.out_port;
3039
3040           old_addr1.as_u32 = ip1->src_address.as_u32;
3041           ip1->src_address.as_u32 = new_addr1.as_u32;
3042           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3043
3044           sum1 = ip1->checksum;
3045           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3046                                  ip4_header_t,
3047                                  src_address /* changed member */);
3048           ip1->checksum = ip_csum_fold (sum1);
3049
3050           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3051             {
3052               if (tcp1->flags & TCP_FLAG_SYN)
3053                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3054               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3055                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3056               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3057                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3058               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3059                 snat_det_ses_close(dm1, ses1);
3060               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3061                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3062               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3063                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3064
3065               old_port1 = tcp1->src;
3066               tcp1->src = new_port1;
3067
3068               sum1 = tcp1->checksum;
3069               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3070                                      ip4_header_t,
3071                                      dst_address /* changed member */);
3072               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3073                                      ip4_header_t /* cheat */,
3074                                      length /* changed member */);
3075               tcp1->checksum = ip_csum_fold(sum1);
3076             }
3077           else
3078             {
3079               ses1->state = SNAT_SESSION_UDP_ACTIVE;
3080               old_port1 = udp1->src_port;
3081               udp1->src_port = new_port1;
3082               udp1->checksum = 0;
3083             }
3084
3085           switch(ses1->state)
3086             {
3087             case SNAT_SESSION_UDP_ACTIVE:
3088                 ses1->expire = now + sm->udp_timeout;
3089                 break;
3090             case SNAT_SESSION_TCP_SYN_SENT:
3091             case SNAT_SESSION_TCP_FIN_WAIT:
3092             case SNAT_SESSION_TCP_CLOSE_WAIT:
3093             case SNAT_SESSION_TCP_LAST_ACK:
3094                 ses1->expire = now + sm->tcp_transitory_timeout;
3095                 break;
3096             case SNAT_SESSION_TCP_ESTABLISHED:
3097                 ses1->expire = now + sm->tcp_established_timeout;
3098                 break;
3099             }
3100
3101         trace1:
3102           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3103                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3104             {
3105               snat_in2out_trace_t *t =
3106                  vlib_add_trace (vm, node, b1, sizeof (*t));
3107               t->is_slow_path = 0;
3108               t->sw_if_index = sw_if_index1;
3109               t->next_index = next1;
3110               t->session_index = ~0;
3111               if (ses1)
3112                 t->session_index = ses1 - dm1->sessions;
3113             }
3114
3115           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3116
3117           /* verify speculative enqueues, maybe switch current next frame */
3118           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3119                                            to_next, n_left_to_next,
3120                                            bi0, bi1, next0, next1);
3121          }
3122
3123       while (n_left_from > 0 && n_left_to_next > 0)
3124         {
3125           u32 bi0;
3126           vlib_buffer_t * b0;
3127           u32 next0;
3128           u32 sw_if_index0;
3129           ip4_header_t * ip0;
3130           ip_csum_t sum0;
3131           ip4_address_t new_addr0, old_addr0;
3132           u16 old_port0, new_port0, lo_port0, i0;
3133           udp_header_t * udp0;
3134           tcp_header_t * tcp0;
3135           u32 proto0;
3136           snat_det_out_key_t key0;
3137           snat_det_map_t * dm0;
3138           snat_det_session_t * ses0 = 0;
3139           u32 rx_fib_index0;
3140           icmp46_header_t * icmp0;
3141
3142           /* speculatively enqueue b0 to the current next frame */
3143           bi0 = from[0];
3144           to_next[0] = bi0;
3145           from += 1;
3146           to_next += 1;
3147           n_left_from -= 1;
3148           n_left_to_next -= 1;
3149
3150           b0 = vlib_get_buffer (vm, bi0);
3151           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3152
3153           ip0 = vlib_buffer_get_current (b0);
3154           udp0 = ip4_next_header (ip0);
3155           tcp0 = (tcp_header_t *) udp0;
3156
3157           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3158
3159           if (PREDICT_FALSE(ip0->ttl == 1))
3160             {
3161               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3162               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3163                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3164                                            0);
3165               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3166               goto trace00;
3167             }
3168
3169           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3170
3171           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3172             {
3173               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3174               icmp0 = (icmp46_header_t *) udp0;
3175
3176               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3177                                   rx_fib_index0, node, next0, thread_index,
3178                                   &ses0, &dm0);
3179               goto trace00;
3180             }
3181
3182           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3183           if (PREDICT_FALSE(!dm0))
3184             {
3185               clib_warning("no match for internal host %U",
3186                            format_ip4_address, &ip0->src_address);
3187               next0 = SNAT_IN2OUT_NEXT_DROP;
3188               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3189               goto trace00;
3190             }
3191
3192           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3193
3194           key0.ext_host_addr = ip0->dst_address;
3195           key0.ext_host_port = tcp0->dst;
3196
3197           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3198           if (PREDICT_FALSE(!ses0))
3199             {
3200               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3201                 {
3202                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3203                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3204
3205                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3206                     continue;
3207
3208                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3209                   break;
3210                 }
3211               if (PREDICT_FALSE(!ses0))
3212                 {
3213                   /* too many sessions for user, send ICMP error packet */
3214
3215                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3216                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3217                                                ICMP4_destination_unreachable_destination_unreachable_host,
3218                                                0);
3219                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3220                   goto trace00;
3221                 }
3222             }
3223
3224           new_port0 = ses0->out.out_port;
3225
3226           old_addr0.as_u32 = ip0->src_address.as_u32;
3227           ip0->src_address.as_u32 = new_addr0.as_u32;
3228           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3229
3230           sum0 = ip0->checksum;
3231           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3232                                  ip4_header_t,
3233                                  src_address /* changed member */);
3234           ip0->checksum = ip_csum_fold (sum0);
3235
3236           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3237             {
3238               if (tcp0->flags & TCP_FLAG_SYN)
3239                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3240               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3241                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3242               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3243                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3244               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3245                 snat_det_ses_close(dm0, ses0);
3246               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3247                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3248               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3249                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3250
3251               old_port0 = tcp0->src;
3252               tcp0->src = new_port0;
3253
3254               sum0 = tcp0->checksum;
3255               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3256                                      ip4_header_t,
3257                                      dst_address /* changed member */);
3258               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3259                                      ip4_header_t /* cheat */,
3260                                      length /* changed member */);
3261               tcp0->checksum = ip_csum_fold(sum0);
3262             }
3263           else
3264             {
3265               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3266               old_port0 = udp0->src_port;
3267               udp0->src_port = new_port0;
3268               udp0->checksum = 0;
3269             }
3270
3271           switch(ses0->state)
3272             {
3273             case SNAT_SESSION_UDP_ACTIVE:
3274                 ses0->expire = now + sm->udp_timeout;
3275                 break;
3276             case SNAT_SESSION_TCP_SYN_SENT:
3277             case SNAT_SESSION_TCP_FIN_WAIT:
3278             case SNAT_SESSION_TCP_CLOSE_WAIT:
3279             case SNAT_SESSION_TCP_LAST_ACK:
3280                 ses0->expire = now + sm->tcp_transitory_timeout;
3281                 break;
3282             case SNAT_SESSION_TCP_ESTABLISHED:
3283                 ses0->expire = now + sm->tcp_established_timeout;
3284                 break;
3285             }
3286
3287         trace00:
3288           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3289                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3290             {
3291               snat_in2out_trace_t *t =
3292                  vlib_add_trace (vm, node, b0, sizeof (*t));
3293               t->is_slow_path = 0;
3294               t->sw_if_index = sw_if_index0;
3295               t->next_index = next0;
3296               t->session_index = ~0;
3297               if (ses0)
3298                 t->session_index = ses0 - dm0->sessions;
3299             }
3300
3301           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3302
3303           /* verify speculative enqueue, maybe switch current next frame */
3304           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3305                                            to_next, n_left_to_next,
3306                                            bi0, next0);
3307         }
3308
3309       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3310     }
3311
3312   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3313                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3314                                pkts_processed);
3315   return frame->n_vectors;
3316 }
3317
3318 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3319   .function = snat_det_in2out_node_fn,
3320   .name = "nat44-det-in2out",
3321   .vector_size = sizeof (u32),
3322   .format_trace = format_snat_in2out_trace,
3323   .type = VLIB_NODE_TYPE_INTERNAL,
3324
3325   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3326   .error_strings = snat_in2out_error_strings,
3327
3328   .runtime_data_bytes = sizeof (snat_runtime_t),
3329
3330   .n_next_nodes = 3,
3331
3332   /* edit / add dispositions here */
3333   .next_nodes = {
3334     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3335     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3336     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3337   },
3338 };
3339
3340 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3341
3342 /**
3343  * Get address and port values to be used for ICMP packet translation
3344  * and create session if needed
3345  *
3346  * @param[in,out] sm             NAT main
3347  * @param[in,out] node           NAT node runtime
3348  * @param[in] thread_index       thread index
3349  * @param[in,out] b0             buffer containing packet to be translated
3350  * @param[out] p_proto           protocol used for matching
3351  * @param[out] p_value           address and port after NAT translation
3352  * @param[out] p_dont_translate  if packet should not be translated
3353  * @param d                      optional parameter
3354  * @param e                      optional parameter
3355  */
3356 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3357                           u32 thread_index, vlib_buffer_t *b0,
3358                           ip4_header_t *ip0, u8 *p_proto,
3359                           snat_session_key_t *p_value,
3360                           u8 *p_dont_translate, void *d, void *e)
3361 {
3362   icmp46_header_t *icmp0;
3363   u32 sw_if_index0;
3364   u32 rx_fib_index0;
3365   u8 protocol;
3366   snat_det_out_key_t key0;
3367   u8 dont_translate = 0;
3368   u32 next0 = ~0;
3369   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3370   ip4_header_t *inner_ip0;
3371   void *l4_header = 0;
3372   icmp46_header_t *inner_icmp0;
3373   snat_det_map_t * dm0 = 0;
3374   ip4_address_t new_addr0;
3375   u16 lo_port0, i0;
3376   snat_det_session_t * ses0 = 0;
3377   ip4_address_t in_addr;
3378   u16 in_port;
3379
3380   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3381   echo0 = (icmp_echo_header_t *)(icmp0+1);
3382   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3383   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3384
3385   if (!icmp_is_error_message (icmp0))
3386     {
3387       protocol = SNAT_PROTOCOL_ICMP;
3388       in_addr = ip0->src_address;
3389       in_port = echo0->identifier;
3390     }
3391   else
3392     {
3393       inner_ip0 = (ip4_header_t *)(echo0+1);
3394       l4_header = ip4_next_header (inner_ip0);
3395       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3396       in_addr = inner_ip0->dst_address;
3397       switch (protocol)
3398         {
3399         case SNAT_PROTOCOL_ICMP:
3400           inner_icmp0 = (icmp46_header_t*)l4_header;
3401           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3402           in_port = inner_echo0->identifier;
3403           break;
3404         case SNAT_PROTOCOL_UDP:
3405         case SNAT_PROTOCOL_TCP:
3406           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3407           break;
3408         default:
3409           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3410           next0 = SNAT_IN2OUT_NEXT_DROP;
3411           goto out;
3412         }
3413     }
3414
3415   dm0 = snat_det_map_by_user(sm, &in_addr);
3416   if (PREDICT_FALSE(!dm0))
3417     {
3418       clib_warning("no match for internal host %U",
3419                    format_ip4_address, &in_addr);
3420       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3421           IP_PROTOCOL_ICMP, rx_fib_index0)))
3422         {
3423           dont_translate = 1;
3424           goto out;
3425         }
3426       next0 = SNAT_IN2OUT_NEXT_DROP;
3427       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3428       goto out;
3429     }
3430
3431   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3432
3433   key0.ext_host_addr = ip0->dst_address;
3434   key0.ext_host_port = 0;
3435
3436   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3437   if (PREDICT_FALSE(!ses0))
3438     {
3439       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3440           IP_PROTOCOL_ICMP, rx_fib_index0)))
3441         {
3442           dont_translate = 1;
3443           goto out;
3444         }
3445       if (icmp0->type != ICMP4_echo_request)
3446         {
3447           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3448           next0 = SNAT_IN2OUT_NEXT_DROP;
3449           goto out;
3450         }
3451       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3452         {
3453           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3454             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3455
3456           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3457             continue;
3458
3459           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3460           break;
3461         }
3462       if (PREDICT_FALSE(!ses0))
3463         {
3464           next0 = SNAT_IN2OUT_NEXT_DROP;
3465           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3466           goto out;
3467         }
3468     }
3469
3470   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3471                     !icmp_is_error_message (icmp0)))
3472     {
3473       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3474       next0 = SNAT_IN2OUT_NEXT_DROP;
3475       goto out;
3476     }
3477
3478   u32 now = (u32) vlib_time_now (sm->vlib_main);
3479
3480   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3481   ses0->expire = now + sm->icmp_timeout;
3482
3483 out:
3484   *p_proto = protocol;
3485   if (ses0)
3486     {
3487       p_value->addr = new_addr0;
3488       p_value->fib_index = sm->outside_fib_index;
3489       p_value->port = ses0->out.out_port;
3490     }
3491   *p_dont_translate = dont_translate;
3492   if (d)
3493     *(snat_det_session_t**)d = ses0;
3494   if (e)
3495     *(snat_det_map_t**)e = dm0;
3496   return next0;
3497 }
3498
3499 /**********************/
3500 /*** worker handoff ***/
3501 /**********************/
3502 static inline uword
3503 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3504                                       vlib_node_runtime_t * node,
3505                                       vlib_frame_t * frame,
3506                                       u8 is_output)
3507 {
3508   snat_main_t *sm = &snat_main;
3509   vlib_thread_main_t *tm = vlib_get_thread_main ();
3510   u32 n_left_from, *from, *to_next = 0;
3511   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3512   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3513     = 0;
3514   vlib_frame_queue_elt_t *hf = 0;
3515   vlib_frame_t *f = 0;
3516   int i;
3517   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3518   u32 next_worker_index = 0;
3519   u32 current_worker_index = ~0;
3520   u32 thread_index = vlib_get_thread_index ();
3521   u32 fq_index;
3522   u32 to_node_index;
3523
3524   ASSERT (vec_len (sm->workers));
3525
3526   if (is_output)
3527     {
3528       fq_index = sm->fq_in2out_output_index;
3529       to_node_index = sm->in2out_output_node_index;
3530     }
3531   else
3532     {
3533       fq_index = sm->fq_in2out_index;
3534       to_node_index = sm->in2out_node_index;
3535     }
3536
3537   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3538     {
3539       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3540
3541       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3542                                sm->first_worker_index + sm->num_workers - 1,
3543                                (vlib_frame_queue_t *) (~0));
3544     }
3545
3546   from = vlib_frame_vector_args (frame);
3547   n_left_from = frame->n_vectors;
3548
3549   while (n_left_from > 0)
3550     {
3551       u32 bi0;
3552       vlib_buffer_t *b0;
3553       u32 sw_if_index0;
3554       u32 rx_fib_index0;
3555       ip4_header_t * ip0;
3556       u8 do_handoff;
3557
3558       bi0 = from[0];
3559       from += 1;
3560       n_left_from -= 1;
3561
3562       b0 = vlib_get_buffer (vm, bi0);
3563
3564       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3565       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3566
3567       ip0 = vlib_buffer_get_current (b0);
3568
3569       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3570
3571       if (PREDICT_FALSE (next_worker_index != thread_index))
3572         {
3573           do_handoff = 1;
3574
3575           if (next_worker_index != current_worker_index)
3576             {
3577               if (hf)
3578                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3579
3580               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3581                                                       next_worker_index,
3582                                                       handoff_queue_elt_by_worker_index);
3583
3584               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3585               to_next_worker = &hf->buffer_index[hf->n_vectors];
3586               current_worker_index = next_worker_index;
3587             }
3588
3589           /* enqueue to correct worker thread */
3590           to_next_worker[0] = bi0;
3591           to_next_worker++;
3592           n_left_to_next_worker--;
3593
3594           if (n_left_to_next_worker == 0)
3595             {
3596               hf->n_vectors = VLIB_FRAME_SIZE;
3597               vlib_put_frame_queue_elt (hf);
3598               current_worker_index = ~0;
3599               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3600               hf = 0;
3601             }
3602         }
3603       else
3604         {
3605           do_handoff = 0;
3606           /* if this is 1st frame */
3607           if (!f)
3608             {
3609               f = vlib_get_frame_to_node (vm, to_node_index);
3610               to_next = vlib_frame_vector_args (f);
3611             }
3612
3613           to_next[0] = bi0;
3614           to_next += 1;
3615           f->n_vectors++;
3616         }
3617
3618       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3619                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3620         {
3621           snat_in2out_worker_handoff_trace_t *t =
3622             vlib_add_trace (vm, node, b0, sizeof (*t));
3623           t->next_worker_index = next_worker_index;
3624           t->do_handoff = do_handoff;
3625         }
3626     }
3627
3628   if (f)
3629     vlib_put_frame_to_node (vm, to_node_index, f);
3630
3631   if (hf)
3632     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3633
3634   /* Ship frames to the worker nodes */
3635   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3636     {
3637       if (handoff_queue_elt_by_worker_index[i])
3638         {
3639           hf = handoff_queue_elt_by_worker_index[i];
3640           /*
3641            * It works better to let the handoff node
3642            * rate-adapt, always ship the handoff queue element.
3643            */
3644           if (1 || hf->n_vectors == hf->last_n_vectors)
3645             {
3646               vlib_put_frame_queue_elt (hf);
3647               handoff_queue_elt_by_worker_index[i] = 0;
3648             }
3649           else
3650             hf->last_n_vectors = hf->n_vectors;
3651         }
3652       congested_handoff_queue_by_worker_index[i] =
3653         (vlib_frame_queue_t *) (~0);
3654     }
3655   hf = 0;
3656   current_worker_index = ~0;
3657   return frame->n_vectors;
3658 }
3659
3660 static uword
3661 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3662                                vlib_node_runtime_t * node,
3663                                vlib_frame_t * frame)
3664 {
3665   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3666 }
3667
3668 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3669   .function = snat_in2out_worker_handoff_fn,
3670   .name = "nat44-in2out-worker-handoff",
3671   .vector_size = sizeof (u32),
3672   .format_trace = format_snat_in2out_worker_handoff_trace,
3673   .type = VLIB_NODE_TYPE_INTERNAL,
3674
3675   .n_next_nodes = 1,
3676
3677   .next_nodes = {
3678     [0] = "error-drop",
3679   },
3680 };
3681
3682 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3683                               snat_in2out_worker_handoff_fn);
3684
3685 static uword
3686 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3687                                       vlib_node_runtime_t * node,
3688                                       vlib_frame_t * frame)
3689 {
3690   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3691 }
3692
3693 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3694   .function = snat_in2out_output_worker_handoff_fn,
3695   .name = "nat44-in2out-output-worker-handoff",
3696   .vector_size = sizeof (u32),
3697   .format_trace = format_snat_in2out_worker_handoff_trace,
3698   .type = VLIB_NODE_TYPE_INTERNAL,
3699
3700   .n_next_nodes = 1,
3701
3702   .next_nodes = {
3703     [0] = "error-drop",
3704   },
3705 };
3706
3707 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3708                               snat_in2out_output_worker_handoff_fn);
3709
3710 static_always_inline int
3711 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3712 {
3713   snat_address_t * ap;
3714   clib_bihash_kv_8_8_t kv, value;
3715   snat_session_key_t m_key;
3716
3717   vec_foreach (ap, sm->addresses)
3718     {
3719       if (ap->addr.as_u32 == dst_addr->as_u32)
3720         return 1;
3721     }
3722
3723   m_key.addr.as_u32 = dst_addr->as_u32;
3724   m_key.fib_index = sm->outside_fib_index;
3725   m_key.port = 0;
3726   m_key.protocol = 0;
3727   kv.key = m_key.as_u64;
3728   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3729     return 1;
3730
3731   return 0;
3732 }
3733
3734 static uword
3735 snat_hairpin_dst_fn (vlib_main_t * vm,
3736                      vlib_node_runtime_t * node,
3737                      vlib_frame_t * frame)
3738 {
3739   u32 n_left_from, * from, * to_next;
3740   snat_in2out_next_t next_index;
3741   u32 pkts_processed = 0;
3742   snat_main_t * sm = &snat_main;
3743
3744   from = vlib_frame_vector_args (frame);
3745   n_left_from = frame->n_vectors;
3746   next_index = node->cached_next_index;
3747
3748   while (n_left_from > 0)
3749     {
3750       u32 n_left_to_next;
3751
3752       vlib_get_next_frame (vm, node, next_index,
3753                            to_next, n_left_to_next);
3754
3755       while (n_left_from > 0 && n_left_to_next > 0)
3756         {
3757           u32 bi0;
3758           vlib_buffer_t * b0;
3759           u32 next0;
3760           ip4_header_t * ip0;
3761           u32 proto0;
3762
3763           /* speculatively enqueue b0 to the current next frame */
3764           bi0 = from[0];
3765           to_next[0] = bi0;
3766           from += 1;
3767           to_next += 1;
3768           n_left_from -= 1;
3769           n_left_to_next -= 1;
3770
3771           b0 = vlib_get_buffer (vm, bi0);
3772           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3773           ip0 = vlib_buffer_get_current (b0);
3774
3775           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3776
3777           vnet_buffer (b0)->snat.flags = 0;
3778           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3779             {
3780               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3781                 {
3782                   udp_header_t * udp0 = ip4_next_header (ip0);
3783                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3784
3785                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3786                 }
3787               else if (proto0 == SNAT_PROTOCOL_ICMP)
3788                 {
3789                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3790
3791                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3792                 }
3793               else
3794                 {
3795                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3796                 }
3797
3798               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3799             }
3800
3801           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3802
3803           /* verify speculative enqueue, maybe switch current next frame */
3804           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3805                                            to_next, n_left_to_next,
3806                                            bi0, next0);
3807          }
3808
3809       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3810     }
3811
3812   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3813                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3814                                pkts_processed);
3815   return frame->n_vectors;
3816 }
3817
3818 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3819   .function = snat_hairpin_dst_fn,
3820   .name = "nat44-hairpin-dst",
3821   .vector_size = sizeof (u32),
3822   .type = VLIB_NODE_TYPE_INTERNAL,
3823   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3824   .error_strings = snat_in2out_error_strings,
3825   .n_next_nodes = 2,
3826   .next_nodes = {
3827     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3828     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3829   },
3830 };
3831
3832 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3833                               snat_hairpin_dst_fn);
3834
3835 static uword
3836 snat_hairpin_src_fn (vlib_main_t * vm,
3837                      vlib_node_runtime_t * node,
3838                      vlib_frame_t * frame)
3839 {
3840   u32 n_left_from, * from, * to_next;
3841   snat_in2out_next_t next_index;
3842   u32 pkts_processed = 0;
3843   snat_main_t *sm = &snat_main;
3844
3845   from = vlib_frame_vector_args (frame);
3846   n_left_from = frame->n_vectors;
3847   next_index = node->cached_next_index;
3848
3849   while (n_left_from > 0)
3850     {
3851       u32 n_left_to_next;
3852
3853       vlib_get_next_frame (vm, node, next_index,
3854                            to_next, n_left_to_next);
3855
3856       while (n_left_from > 0 && n_left_to_next > 0)
3857         {
3858           u32 bi0;
3859           vlib_buffer_t * b0;
3860           u32 next0;
3861           snat_interface_t *i;
3862           u32 sw_if_index0;
3863
3864           /* speculatively enqueue b0 to the current next frame */
3865           bi0 = from[0];
3866           to_next[0] = bi0;
3867           from += 1;
3868           to_next += 1;
3869           n_left_from -= 1;
3870           n_left_to_next -= 1;
3871
3872           b0 = vlib_get_buffer (vm, bi0);
3873           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3874           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3875
3876           pool_foreach (i, sm->output_feature_interfaces,
3877           ({
3878             /* Only packets from NAT inside interface */
3879             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3880               {
3881                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3882                                     SNAT_FLAG_HAIRPINNING))
3883                   {
3884                     if (PREDICT_TRUE (sm->num_workers > 1))
3885                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3886                     else
3887                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3888                   }
3889                 break;
3890               }
3891           }));
3892
3893           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3894
3895           /* verify speculative enqueue, maybe switch current next frame */
3896           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3897                                            to_next, n_left_to_next,
3898                                            bi0, next0);
3899          }
3900
3901       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3902     }
3903
3904   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3905                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3906                                pkts_processed);
3907   return frame->n_vectors;
3908 }
3909
3910 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3911   .function = snat_hairpin_src_fn,
3912   .name = "nat44-hairpin-src",
3913   .vector_size = sizeof (u32),
3914   .type = VLIB_NODE_TYPE_INTERNAL,
3915   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3916   .error_strings = snat_in2out_error_strings,
3917   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3918   .next_nodes = {
3919      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3920      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3921      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3922      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3923   },
3924 };
3925
3926 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3927                               snat_hairpin_src_fn);
3928
3929 static uword
3930 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3931                                 vlib_node_runtime_t * node,
3932                                 vlib_frame_t * frame)
3933 {
3934   u32 n_left_from, * from, * to_next;
3935   snat_in2out_next_t next_index;
3936   u32 pkts_processed = 0;
3937   snat_main_t * sm = &snat_main;
3938   u32 stats_node_index;
3939
3940   stats_node_index = snat_in2out_fast_node.index;
3941
3942   from = vlib_frame_vector_args (frame);
3943   n_left_from = frame->n_vectors;
3944   next_index = node->cached_next_index;
3945
3946   while (n_left_from > 0)
3947     {
3948       u32 n_left_to_next;
3949
3950       vlib_get_next_frame (vm, node, next_index,
3951                            to_next, n_left_to_next);
3952
3953       while (n_left_from > 0 && n_left_to_next > 0)
3954         {
3955           u32 bi0;
3956           vlib_buffer_t * b0;
3957           u32 next0;
3958           u32 sw_if_index0;
3959           ip4_header_t * ip0;
3960           ip_csum_t sum0;
3961           u32 new_addr0, old_addr0;
3962           u16 old_port0, new_port0;
3963           udp_header_t * udp0;
3964           tcp_header_t * tcp0;
3965           icmp46_header_t * icmp0;
3966           snat_session_key_t key0, sm0;
3967           u32 proto0;
3968           u32 rx_fib_index0;
3969
3970           /* speculatively enqueue b0 to the current next frame */
3971           bi0 = from[0];
3972           to_next[0] = bi0;
3973           from += 1;
3974           to_next += 1;
3975           n_left_from -= 1;
3976           n_left_to_next -= 1;
3977
3978           b0 = vlib_get_buffer (vm, bi0);
3979           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3980
3981           ip0 = vlib_buffer_get_current (b0);
3982           udp0 = ip4_next_header (ip0);
3983           tcp0 = (tcp_header_t *) udp0;
3984           icmp0 = (icmp46_header_t *) udp0;
3985
3986           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3987           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3988
3989           if (PREDICT_FALSE(ip0->ttl == 1))
3990             {
3991               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3992               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3993                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3994                                            0);
3995               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3996               goto trace0;
3997             }
3998
3999           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4000
4001           if (PREDICT_FALSE (proto0 == ~0))
4002               goto trace0;
4003
4004           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4005             {
4006               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4007                                   rx_fib_index0, node, next0, ~0, 0, 0);
4008               goto trace0;
4009             }
4010
4011           key0.addr = ip0->src_address;
4012           key0.protocol = proto0;
4013           key0.port = udp0->src_port;
4014           key0.fib_index = rx_fib_index0;
4015
4016           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0))
4017             {
4018               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4019               next0= SNAT_IN2OUT_NEXT_DROP;
4020               goto trace0;
4021             }
4022
4023           new_addr0 = sm0.addr.as_u32;
4024           new_port0 = sm0.port;
4025           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4026           old_addr0 = ip0->src_address.as_u32;
4027           ip0->src_address.as_u32 = new_addr0;
4028
4029           sum0 = ip0->checksum;
4030           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4031                                  ip4_header_t,
4032                                  src_address /* changed member */);
4033           ip0->checksum = ip_csum_fold (sum0);
4034
4035           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4036             {
4037               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4038                 {
4039                   old_port0 = tcp0->src_port;
4040                   tcp0->src_port = new_port0;
4041
4042                   sum0 = tcp0->checksum;
4043                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4044                                          ip4_header_t,
4045                                          dst_address /* changed member */);
4046                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4047                                          ip4_header_t /* cheat */,
4048                                          length /* changed member */);
4049                   tcp0->checksum = ip_csum_fold(sum0);
4050                 }
4051               else
4052                 {
4053                   old_port0 = udp0->src_port;
4054                   udp0->src_port = new_port0;
4055                   udp0->checksum = 0;
4056                 }
4057             }
4058           else
4059             {
4060               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4061                 {
4062                   sum0 = tcp0->checksum;
4063                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4064                                          ip4_header_t,
4065                                          dst_address /* changed member */);
4066                   tcp0->checksum = ip_csum_fold(sum0);
4067                 }
4068             }
4069
4070           /* Hairpinning */
4071           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4072
4073         trace0:
4074           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4075                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4076             {
4077               snat_in2out_trace_t *t =
4078                  vlib_add_trace (vm, node, b0, sizeof (*t));
4079               t->sw_if_index = sw_if_index0;
4080               t->next_index = next0;
4081             }
4082
4083           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4084
4085           /* verify speculative enqueue, maybe switch current next frame */
4086           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4087                                            to_next, n_left_to_next,
4088                                            bi0, next0);
4089         }
4090
4091       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4092     }
4093
4094   vlib_node_increment_counter (vm, stats_node_index,
4095                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4096                                pkts_processed);
4097   return frame->n_vectors;
4098 }
4099
4100
4101 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
4102   .function = snat_in2out_fast_static_map_fn,
4103   .name = "nat44-in2out-fast",
4104   .vector_size = sizeof (u32),
4105   .format_trace = format_snat_in2out_fast_trace,
4106   .type = VLIB_NODE_TYPE_INTERNAL,
4107
4108   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4109   .error_strings = snat_in2out_error_strings,
4110
4111   .runtime_data_bytes = sizeof (snat_runtime_t),
4112
4113   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4114
4115   /* edit / add dispositions here */
4116   .next_nodes = {
4117     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4118     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4119     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4120     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4121     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4122   },
4123 };
4124
4125 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);