src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37   u32 is_slow_path;
  38 } snat_in2out_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_in2out_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  51   char * tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  66
  67   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  68               t->sw_if_index, t->next_index);
  69
  70   return s;
  71 }
  72
  73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   snat_in2out_worker_handoff_trace_t * t =
  78     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  79   char * m;
  80
  81   m = t->do_handoff ? "next worker" : "same worker";
  82   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  83
  84   return s;
  85 }
  86
  87 typedef struct {
  88   u32 sw_if_index;
  89   u32 next_index;
  90   u8 cached;
  91 } nat44_in2out_reass_trace_t;
  92
  93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  94 {
  95   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  96   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  97   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  98
  99   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 100               t->sw_if_index, t->next_index,
 101               t->cached ? "cached" : "translated");
 102
 103   return s;
 104 }
 105
 106 vlib_node_registration_t snat_in2out_node;
 107 vlib_node_registration_t snat_in2out_slowpath_node;
 108 vlib_node_registration_t snat_in2out_fast_node;
 109 vlib_node_registration_t snat_in2out_worker_handoff_node;
 110 vlib_node_registration_t snat_det_in2out_node;
 111 vlib_node_registration_t snat_in2out_output_node;
 112 vlib_node_registration_t snat_in2out_output_slowpath_node;
 113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 114 vlib_node_registration_t snat_hairpin_dst_node;
 115 vlib_node_registration_t snat_hairpin_src_node;
 116 vlib_node_registration_t nat44_hairpinning_node;
 117 vlib_node_registration_t nat44_in2out_reass_node;
 118
 119
 120 #define foreach_snat_in2out_error                       \
 121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 122 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 123 _(OUT_OF_PORTS, "Out of ports")                         \
 124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 125 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 126 _(NO_TRANSLATION, "No translation")                     \
 127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 128 _(DROP_FRAGMENT, "Drop fragment")                       \
 129 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 131
 132 typedef enum {
 133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 134   foreach_snat_in2out_error
 135 #undef _
 136   SNAT_IN2OUT_N_ERROR,
 137 } snat_in2out_error_t;
 138
 139 static char * snat_in2out_error_strings[] = {
 140 #define _(sym,string) string,
 141   foreach_snat_in2out_error
 142 #undef _
 143 };
 144
 145 typedef enum {
 146   SNAT_IN2OUT_NEXT_LOOKUP,
 147   SNAT_IN2OUT_NEXT_DROP,
 148   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 149   SNAT_IN2OUT_NEXT_SLOW_PATH,
 150   SNAT_IN2OUT_NEXT_REASS,
 151   SNAT_IN2OUT_N_NEXT,
 152 } snat_in2out_next_t;
 153
 154 typedef enum {
 155   SNAT_HAIRPIN_SRC_NEXT_DROP,
 156   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 157   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 158   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 159   SNAT_HAIRPIN_SRC_N_NEXT,
 160 } snat_hairpin_next_t;
 161
 162 /**
 163  * @brief Check if packet should be translated
 164  *
 165  * Packets aimed at outside interface and external address with active session
 166  * should be translated.
 167  *
 168  * @param sm            NAT main
 169  * @param rt            NAT runtime data
 170  * @param sw_if_index0  index of the inside interface
 171  * @param ip0           IPv4 header
 172  * @param proto0        NAT protocol
 173  * @param rx_fib_index0 RX FIB index
 174  *
 175  * @returns 0 if packet should be translated otherwise 1
 176  */
 177 static inline int
 178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 179                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                          u32 rx_fib_index0)
 181 {
 182   if (sm->out2in_dpo)
 183     return 0;
 184
 185   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 186   fib_prefix_t pfx = {
 187     .fp_proto = FIB_PROTOCOL_IP4,
 188     .fp_len = 32,
 189     .fp_addr = {
 190         .ip4.as_u32 = ip0->dst_address.as_u32,
 191     },
 192   };
 193
 194   /* Don't NAT packet aimed at the intfc address */
 195   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 196                                       ip0->dst_address.as_u32)))
 197     return 1;
 198
 199   fei = fib_table_lookup (rx_fib_index0, &pfx);
 200   if (FIB_NODE_INDEX_INVALID != fei)
 201     {
 202       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 203       if (sw_if_index == ~0)
 204         {
 205           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 206           if (FIB_NODE_INDEX_INVALID != fei)
 207             sw_if_index = fib_entry_get_resolving_interface (fei);
 208         }
 209       snat_interface_t *i;
 210       pool_foreach (i, sm->interfaces,
 211       ({
 212         /* NAT packet aimed at outside interface */
 213         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 214           return 0;
 215       }));
 216     }
 217
 218   return 1;
 219 }
 220
 221 static inline int
 222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 223                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 224                     u32 rx_fib_index0, u32 thread_index)
 225 {
 226   udp_header_t * udp0 = ip4_next_header (ip0);
 227   snat_session_key_t key0, sm0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229
 230   key0.addr = ip0->dst_address;
 231   key0.port = udp0->dst_port;
 232   key0.protocol = proto0;
 233   key0.fib_index = sm->outside_fib_index;
 234   kv0.key = key0.as_u64;
 235
 236   /* NAT packet aimed at external address if */
 237   /* has active sessions */
 238   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 239                               &value0))
 240     {
 241       /* or is static mappings */
 242       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 243         return 0;
 244     }
 245   else
 246     return 0;
 247
 248   if (sm->forwarding_enabled)
 249     return 1;
 250
 251   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 252                                  rx_fib_index0);
 253 }
 254
 255 static inline int
 256 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 257                                   u32 proto0, u16 src_port, u16 dst_port,
 258                                   u32 thread_index, u32 sw_if_index)
 259 {
 260   snat_session_key_t key0;
 261   clib_bihash_kv_8_8_t kv0, value0;
 262   snat_interface_t *i;
 263
 264   /* src NAT check */
 265   key0.addr = ip0->src_address;
 266   key0.port = src_port;
 267   key0.protocol = proto0;
 268   key0.fib_index = sm->outside_fib_index;
 269   kv0.key = key0.as_u64;
 270
 271   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 272                                &value0))
 273     return 1;
 274
 275   /* dst NAT check */
 276   key0.addr = ip0->dst_address;
 277   key0.port = dst_port;
 278   key0.protocol = proto0;
 279   key0.fib_index = sm->inside_fib_index;
 280   kv0.key = key0.as_u64;
 281   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 282                                &value0))
 283   {
 284     /* hairpinning */
 285     pool_foreach (i, sm->output_feature_interfaces,
 286     ({
 287       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 288         return 0;
 289     }));
 290     return 1;
 291   }
 292
 293   return 0;
 294 }
 295
 296 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 297                       ip4_header_t * ip0,
 298                       u32 rx_fib_index0,
 299                       snat_session_key_t * key0,
 300                       snat_session_t ** sessionp,
 301                       vlib_node_runtime_t * node,
 302                       u32 next0,
 303                       u32 thread_index)
 304 {
 305   snat_user_t *u;
 306   snat_session_t *s;
 307   clib_bihash_kv_8_8_t kv0;
 308   snat_session_key_t key1;
 309   u32 address_index = ~0;
 310   u32 outside_fib_index;
 311   uword * p;
 312   udp_header_t * udp0 = ip4_next_header (ip0);
 313   u8 is_sm = 0;
 314
 315   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 316     {
 317       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 318       nat_ipfix_logging_max_sessions(sm->max_translations);
 319       return SNAT_IN2OUT_NEXT_DROP;
 320     }
 321
 322   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 323   if (! p)
 324     {
 325       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 326       return SNAT_IN2OUT_NEXT_DROP;
 327     }
 328   outside_fib_index = p[0];
 329
 330   key1.protocol = key0->protocol;
 331
 332   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 333                               thread_index);
 334   if (!u)
 335     {
 336       clib_warning ("create NAT user failed");
 337       return SNAT_IN2OUT_NEXT_DROP;
 338     }
 339
 340   /* First try to match static mapping by local address and port */
 341   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
 342     {
 343       /* Try to create dynamic translation */
 344       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 345                                                thread_index, &key1,
 346                                                &address_index,
 347                                                sm->port_per_thread,
 348                                                sm->per_thread_data[thread_index].snat_thread_index))
 349         {
 350           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 351           return SNAT_IN2OUT_NEXT_DROP;
 352         }
 353     }
 354   else
 355     is_sm = 1;
 356
 357   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 358   if (!s)
 359     {
 360       clib_warning ("create NAT session failed");
 361       return SNAT_IN2OUT_NEXT_DROP;
 362     }
 363
 364   if (is_sm)
 365     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 366   user_session_increment (sm, u, is_sm);
 367   s->outside_address_index = address_index;
 368   s->in2out = *key0;
 369   s->out2in = key1;
 370   s->out2in.protocol = key0->protocol;
 371   s->out2in.fib_index = outside_fib_index;
 372   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 373   s->ext_host_port = udp0->dst_port;
 374   *sessionp = s;
 375
 376   /* Add to translation hashes */
 377   kv0.key = s->in2out.as_u64;
 378   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 379   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 380                                1 /* is_add */))
 381       clib_warning ("in2out key add failed");
 382
 383   kv0.key = s->out2in.as_u64;
 384   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 385
 386   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 387                                1 /* is_add */))
 388       clib_warning ("out2in key add failed");
 389
 390   /* log NAT event */
 391   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 392                                       s->out2in.addr.as_u32,
 393                                       s->in2out.protocol,
 394                                       s->in2out.port,
 395                                       s->out2in.port,
 396                                       s->in2out.fib_index);
 397   return next0;
 398 }
 399
 400 static_always_inline
 401 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 402                                  snat_session_key_t *p_key0)
 403 {
 404   icmp46_header_t *icmp0;
 405   snat_session_key_t key0;
 406   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 407   ip4_header_t *inner_ip0 = 0;
 408   void *l4_header = 0;
 409   icmp46_header_t *inner_icmp0;
 410
 411   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 412   echo0 = (icmp_echo_header_t *)(icmp0+1);
 413
 414   if (!icmp_is_error_message (icmp0))
 415     {
 416       key0.protocol = SNAT_PROTOCOL_ICMP;
 417       key0.addr = ip0->src_address;
 418       key0.port = echo0->identifier;
 419     }
 420   else
 421     {
 422       inner_ip0 = (ip4_header_t *)(echo0+1);
 423       l4_header = ip4_next_header (inner_ip0);
 424       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 425       key0.addr = inner_ip0->dst_address;
 426       switch (key0.protocol)
 427         {
 428         case SNAT_PROTOCOL_ICMP:
 429           inner_icmp0 = (icmp46_header_t*)l4_header;
 430           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 431           key0.port = inner_echo0->identifier;
 432           break;
 433         case SNAT_PROTOCOL_UDP:
 434         case SNAT_PROTOCOL_TCP:
 435           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 436           break;
 437         default:
 438           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 439         }
 440     }
 441   *p_key0 = key0;
 442   return -1; /* success */
 443 }
 444
 445 static_always_inline int
 446 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 447 {
 448   icmp46_header_t *icmp0;
 449   nat_ed_ses_key_t key0;
 450   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 451   ip4_header_t *inner_ip0 = 0;
 452   void *l4_header = 0;
 453   icmp46_header_t *inner_icmp0;
 454
 455   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 456   echo0 = (icmp_echo_header_t *)(icmp0+1);
 457
 458   if (!icmp_is_error_message (icmp0))
 459     {
 460       key0.proto = IP_PROTOCOL_ICMP;
 461       key0.l_addr = ip0->src_address;
 462       key0.r_addr = ip0->dst_address;
 463       key0.l_port = key0.r_port = echo0->identifier;
 464     }
 465   else
 466     {
 467       inner_ip0 = (ip4_header_t *)(echo0+1);
 468       l4_header = ip4_next_header (inner_ip0);
 469       key0.proto = inner_ip0->protocol;
 470       key0.r_addr = inner_ip0->src_address;
 471       key0.l_addr = inner_ip0->dst_address;
 472       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 473         {
 474         case SNAT_PROTOCOL_ICMP:
 475           inner_icmp0 = (icmp46_header_t*)l4_header;
 476           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 477           key0.r_port = key0.l_port = inner_echo0->identifier;
 478           break;
 479         case SNAT_PROTOCOL_UDP:
 480         case SNAT_PROTOCOL_TCP:
 481           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 482           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
 483           break;
 484         default:
 485           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 486         }
 487     }
 488   *p_key0 = key0;
 489   return 0;
 490 }
 491
 492 static inline int
 493 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 494                                       u32 thread_index)
 495 {
 496   nat_ed_ses_key_t key;
 497   clib_bihash_kv_16_8_t kv, value;
 498   udp_header_t *udp;
 499   snat_session_t *s = 0;
 500   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 501
 502   if (!sm->forwarding_enabled)
 503     return 0;
 504
 505   if (ip->protocol == IP_PROTOCOL_ICMP)
 506     {
 507       if (icmp_get_ed_key (ip, &key))
 508         return 0;
 509     }
 510   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 511     {
 512       udp = ip4_next_header(ip);
 513       key.l_addr = ip->src_address;
 514       key.r_addr = ip->dst_address;
 515       key.proto = ip->protocol;
 516       key.r_port = udp->dst_port;
 517       key.l_port = udp->src_port;
 518     }
 519   else
 520     {
 521       key.l_addr = ip->src_address;
 522       key.r_addr = ip->dst_address;
 523       key.proto = ip->protocol;
 524       key.l_port = key.r_port = 0;
 525     }
 526   key.fib_index = 0;
 527   kv.key[0] = key.as_u64[0];
 528   kv.key[1] = key.as_u64[1];
 529
 530   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 531     {
 532       s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value.value);
 533       if (is_fwd_bypass_session (s))
 534         {
 535           /* Per-user LRU list maintenance */
 536           clib_dlist_remove (tsm->list_pool, s->per_user_index);
 537           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 538                               s->per_user_index);
 539           return 1;
 540         }
 541       else
 542         return 0;
 543     }
 544
 545   return 0;
 546 }
 547
 548 /**
 549  * Get address and port values to be used for ICMP packet translation
 550  * and create session if needed
 551  *
 552  * @param[in,out] sm             NAT main
 553  * @param[in,out] node           NAT node runtime
 554  * @param[in] thread_index       thread index
 555  * @param[in,out] b0             buffer containing packet to be translated
 556  * @param[out] p_proto           protocol used for matching
 557  * @param[out] p_value           address and port after NAT translation
 558  * @param[out] p_dont_translate  if packet should not be translated
 559  * @param d                      optional parameter
 560  * @param e                      optional parameter
 561  */
 562 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 563                            u32 thread_index, vlib_buffer_t *b0,
 564                            ip4_header_t *ip0, u8 *p_proto,
 565                            snat_session_key_t *p_value,
 566                            u8 *p_dont_translate, void *d, void *e)
 567 {
 568   icmp46_header_t *icmp0;
 569   u32 sw_if_index0;
 570   u32 rx_fib_index0;
 571   snat_session_key_t key0;
 572   snat_session_t *s0 = 0;
 573   u8 dont_translate = 0;
 574   clib_bihash_kv_8_8_t kv0, value0;
 575   u32 next0 = ~0;
 576   int err;
 577
 578   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 579   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 580   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 581
 582   err = icmp_get_key (ip0, &key0);
 583   if (err != -1)
 584     {
 585       b0->error = node->errors[err];
 586       next0 = SNAT_IN2OUT_NEXT_DROP;
 587       goto out;
 588     }
 589   key0.fib_index = rx_fib_index0;
 590
 591   kv0.key = key0.as_u64;
 592
 593   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 594                               &value0))
 595     {
 596       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 597         {
 598           if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
 599               ip0, SNAT_PROTOCOL_ICMP, key0.port, key0.port, thread_index, sw_if_index0)))
 600             {
 601               dont_translate = 1;
 602               goto out;
 603             }
 604         }
 605       else
 606         {
 607           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 608               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 609             {
 610               dont_translate = 1;
 611               goto out;
 612             }
 613         }
 614
 615       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 616         {
 617           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 618           next0 = SNAT_IN2OUT_NEXT_DROP;
 619           goto out;
 620         }
 621
 622       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 623                          &s0, node, next0, thread_index);
 624
 625       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 626         goto out;
 627     }
 628   else
 629     {
 630       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 631                         icmp0->type != ICMP4_echo_reply &&
 632                         !icmp_is_error_message (icmp0)))
 633         {
 634           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 635           next0 = SNAT_IN2OUT_NEXT_DROP;
 636           goto out;
 637         }
 638
 639       if (PREDICT_FALSE (value0.value == ~0ULL))
 640         {
 641           nat_ed_ses_key_t key;
 642           clib_bihash_kv_16_8_t s_kv, s_value;
 643
 644           key.as_u64[0] = 0;
 645           key.as_u64[1] = 0;
 646           if (icmp_get_ed_key (ip0, &key))
 647             {
 648               b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 649               next0 = SNAT_IN2OUT_NEXT_DROP;
 650               goto out;
 651             }
 652           key.fib_index = rx_fib_index0;
 653           s_kv.key[0] = key.as_u64[0];
 654           s_kv.key[1] = key.as_u64[1];
 655           if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 656             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 657                                     s_value.value);
 658           else
 659            {
 660               next0 = SNAT_IN2OUT_NEXT_DROP;
 661               goto out;
 662            }
 663         }
 664       else
 665         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 666                                 value0.value);
 667     }
 668
 669 out:
 670   *p_proto = key0.protocol;
 671   if (s0)
 672     *p_value = s0->out2in;
 673   *p_dont_translate = dont_translate;
 674   if (d)
 675     *(snat_session_t**)d = s0;
 676   return next0;
 677 }
 678
 679 /**
 680  * Get address and port values to be used for ICMP packet translation
 681  *
 682  * @param[in] sm                 NAT main
 683  * @param[in,out] node           NAT node runtime
 684  * @param[in] thread_index       thread index
 685  * @param[in,out] b0             buffer containing packet to be translated
 686  * @param[out] p_proto           protocol used for matching
 687  * @param[out] p_value           address and port after NAT translation
 688  * @param[out] p_dont_translate  if packet should not be translated
 689  * @param d                      optional parameter
 690  * @param e                      optional parameter
 691  */
 692 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 693                            u32 thread_index, vlib_buffer_t *b0,
 694                            ip4_header_t *ip0, u8 *p_proto,
 695                            snat_session_key_t *p_value,
 696                            u8 *p_dont_translate, void *d, void *e)
 697 {
 698   icmp46_header_t *icmp0;
 699   u32 sw_if_index0;
 700   u32 rx_fib_index0;
 701   snat_session_key_t key0;
 702   snat_session_key_t sm0;
 703   u8 dont_translate = 0;
 704   u8 is_addr_only;
 705   u32 next0 = ~0;
 706   int err;
 707
 708   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 709   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 710   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 711
 712   err = icmp_get_key (ip0, &key0);
 713   if (err != -1)
 714     {
 715       b0->error = node->errors[err];
 716       next0 = SNAT_IN2OUT_NEXT_DROP;
 717       goto out2;
 718     }
 719   key0.fib_index = rx_fib_index0;
 720
 721   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
 722     {
 723       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 724           IP_PROTOCOL_ICMP, rx_fib_index0)))
 725         {
 726           dont_translate = 1;
 727           goto out;
 728         }
 729
 730       if (icmp_is_error_message (icmp0))
 731         {
 732           next0 = SNAT_IN2OUT_NEXT_DROP;
 733           goto out;
 734         }
 735
 736       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 737       next0 = SNAT_IN2OUT_NEXT_DROP;
 738       goto out;
 739     }
 740
 741   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 742                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 743                     !icmp_is_error_message (icmp0)))
 744     {
 745       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 746       next0 = SNAT_IN2OUT_NEXT_DROP;
 747       goto out;
 748     }
 749
 750 out:
 751   *p_value = sm0;
 752 out2:
 753   *p_proto = key0.protocol;
 754   *p_dont_translate = dont_translate;
 755   return next0;
 756 }
 757
 758 static inline u32 icmp_in2out (snat_main_t *sm,
 759                                vlib_buffer_t * b0,
 760                                ip4_header_t * ip0,
 761                                icmp46_header_t * icmp0,
 762                                u32 sw_if_index0,
 763                                u32 rx_fib_index0,
 764                                vlib_node_runtime_t * node,
 765                                u32 next0,
 766                                u32 thread_index,
 767                                void *d,
 768                                void *e)
 769 {
 770   snat_session_key_t sm0;
 771   u8 protocol;
 772   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 773   ip4_header_t *inner_ip0;
 774   void *l4_header = 0;
 775   icmp46_header_t *inner_icmp0;
 776   u8 dont_translate;
 777   u32 new_addr0, old_addr0;
 778   u16 old_id0, new_id0;
 779   ip_csum_t sum0;
 780   u16 checksum0;
 781   u32 next0_tmp;
 782
 783   echo0 = (icmp_echo_header_t *)(icmp0+1);
 784
 785   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 786                                        &protocol, &sm0, &dont_translate, d, e);
 787   if (next0_tmp != ~0)
 788     next0 = next0_tmp;
 789   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 790     goto out;
 791
 792   sum0 = ip_incremental_checksum (0, icmp0,
 793                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 794   checksum0 = ~ip_csum_fold (sum0);
 795   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 796     {
 797       next0 = SNAT_IN2OUT_NEXT_DROP;
 798       goto out;
 799     }
 800
 801   old_addr0 = ip0->src_address.as_u32;
 802   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 803   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 804     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 805
 806   sum0 = ip0->checksum;
 807   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 808                          src_address /* changed member */);
 809   ip0->checksum = ip_csum_fold (sum0);
 810
 811   if (icmp0->checksum == 0)
 812     icmp0->checksum = 0xffff;
 813
 814   if (!icmp_is_error_message (icmp0))
 815     {
 816       new_id0 = sm0.port;
 817       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 818         {
 819           old_id0 = echo0->identifier;
 820           new_id0 = sm0.port;
 821           echo0->identifier = new_id0;
 822
 823           sum0 = icmp0->checksum;
 824           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 825                                  identifier);
 826           icmp0->checksum = ip_csum_fold (sum0);
 827         }
 828     }
 829   else
 830     {
 831       inner_ip0 = (ip4_header_t *)(echo0+1);
 832       l4_header = ip4_next_header (inner_ip0);
 833
 834       if (!ip4_header_checksum_is_valid (inner_ip0))
 835         {
 836           next0 = SNAT_IN2OUT_NEXT_DROP;
 837           goto out;
 838         }
 839
 840       old_addr0 = inner_ip0->dst_address.as_u32;
 841       inner_ip0->dst_address = sm0.addr;
 842       new_addr0 = inner_ip0->dst_address.as_u32;
 843
 844       sum0 = icmp0->checksum;
 845       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 846                              dst_address /* changed member */);
 847       icmp0->checksum = ip_csum_fold (sum0);
 848
 849       switch (protocol)
 850         {
 851           case SNAT_PROTOCOL_ICMP:
 852             inner_icmp0 = (icmp46_header_t*)l4_header;
 853             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 854
 855             old_id0 = inner_echo0->identifier;
 856             new_id0 = sm0.port;
 857             inner_echo0->identifier = new_id0;
 858
 859             sum0 = icmp0->checksum;
 860             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 861                                    identifier);
 862             icmp0->checksum = ip_csum_fold (sum0);
 863             break;
 864           case SNAT_PROTOCOL_UDP:
 865           case SNAT_PROTOCOL_TCP:
 866             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 867             new_id0 = sm0.port;
 868             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 869
 870             sum0 = icmp0->checksum;
 871             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 872                                    dst_port);
 873             icmp0->checksum = ip_csum_fold (sum0);
 874             break;
 875           default:
 876             ASSERT(0);
 877         }
 878     }
 879
 880 out:
 881   return next0;
 882 }
 883
 884 /**
 885  * @brief Hairpinning
 886  *
 887  * Hairpinning allows two endpoints on the internal side of the NAT to
 888  * communicate even if they only use each other's external IP addresses
 889  * and ports.
 890  *
 891  * @param sm     NAT main.
 892  * @param b0     Vlib buffer.
 893  * @param ip0    IP header.
 894  * @param udp0   UDP header.
 895  * @param tcp0   TCP header.
 896  * @param proto0 NAT protocol.
 897  */
 898 static inline int
 899 snat_hairpinning (snat_main_t *sm,
 900                   vlib_buffer_t * b0,
 901                   ip4_header_t * ip0,
 902                   udp_header_t * udp0,
 903                   tcp_header_t * tcp0,
 904                   u32 proto0)
 905 {
 906   snat_session_key_t key0, sm0;
 907   snat_session_t * s0;
 908   clib_bihash_kv_8_8_t kv0, value0;
 909   ip_csum_t sum0;
 910   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 911   u16 new_dst_port0, old_dst_port0;
 912
 913   key0.addr = ip0->dst_address;
 914   key0.port = udp0->dst_port;
 915   key0.protocol = proto0;
 916   key0.fib_index = sm->outside_fib_index;
 917   kv0.key = key0.as_u64;
 918
 919   /* Check if destination is static mappings */
 920   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 921     {
 922       new_dst_addr0 = sm0.addr.as_u32;
 923       new_dst_port0 = sm0.port;
 924       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 925     }
 926   /* or active session */
 927   else
 928     {
 929       if (sm->num_workers > 1)
 930         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 931       else
 932         ti = sm->num_workers;
 933
 934       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 935         {
 936           si = value0.value;
 937
 938           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 939           new_dst_addr0 = s0->in2out.addr.as_u32;
 940           new_dst_port0 = s0->in2out.port;
 941           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 942         }
 943     }
 944
 945   /* Destination is behind the same NAT, use internal address and port */
 946   if (new_dst_addr0)
 947     {
 948       old_dst_addr0 = ip0->dst_address.as_u32;
 949       ip0->dst_address.as_u32 = new_dst_addr0;
 950       sum0 = ip0->checksum;
 951       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 952                              ip4_header_t, dst_address);
 953       ip0->checksum = ip_csum_fold (sum0);
 954
 955       old_dst_port0 = tcp0->dst;
 956       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 957         {
 958           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 959             {
 960               tcp0->dst = new_dst_port0;
 961               sum0 = tcp0->checksum;
 962               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 963                                      ip4_header_t, dst_address);
 964               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 965                                      ip4_header_t /* cheat */, length);
 966               tcp0->checksum = ip_csum_fold(sum0);
 967             }
 968           else
 969             {
 970               udp0->dst_port = new_dst_port0;
 971               udp0->checksum = 0;
 972             }
 973         }
 974       else
 975         {
 976           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 977             {
 978               sum0 = tcp0->checksum;
 979               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 980                                      ip4_header_t, dst_address);
 981               tcp0->checksum = ip_csum_fold(sum0);
 982             }
 983         }
 984       return 1;
 985     }
 986   return 0;
 987 }
 988
 989 static inline void
 990 snat_icmp_hairpinning (snat_main_t *sm,
 991                        vlib_buffer_t * b0,
 992                        ip4_header_t * ip0,
 993                        icmp46_header_t * icmp0)
 994 {
 995   snat_session_key_t key0, sm0;
 996   clib_bihash_kv_8_8_t kv0, value0;
 997   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 998   ip_csum_t sum0;
 999   snat_session_t *s0;
1000
1001   if (!icmp_is_error_message (icmp0))
1002     {
1003       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
1004       u16 icmp_id0 = echo0->identifier;
1005       key0.addr = ip0->dst_address;
1006       key0.port = icmp_id0;
1007       key0.protocol = SNAT_PROTOCOL_ICMP;
1008       key0.fib_index = sm->outside_fib_index;
1009       kv0.key = key0.as_u64;
1010
1011       if (sm->num_workers > 1)
1012         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
1013       else
1014         ti = sm->num_workers;
1015
1016       /* Check if destination is in active sessions */
1017       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1018                                   &value0))
1019         {
1020           /* or static mappings */
1021           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1022             {
1023               new_dst_addr0 = sm0.addr.as_u32;
1024               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1025             }
1026         }
1027       else
1028         {
1029           si = value0.value;
1030
1031           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1032           new_dst_addr0 = s0->in2out.addr.as_u32;
1033           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1034           echo0->identifier = s0->in2out.port;
1035           sum0 = icmp0->checksum;
1036           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1037                                  icmp_echo_header_t, identifier);
1038           icmp0->checksum = ip_csum_fold (sum0);
1039         }
1040
1041       /* Destination is behind the same NAT, use internal address and port */
1042       if (new_dst_addr0)
1043         {
1044           old_dst_addr0 = ip0->dst_address.as_u32;
1045           ip0->dst_address.as_u32 = new_dst_addr0;
1046           sum0 = ip0->checksum;
1047           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1048                                  ip4_header_t, dst_address);
1049           ip0->checksum = ip_csum_fold (sum0);
1050         }
1051     }
1052
1053 }
1054
1055 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1056                                          vlib_buffer_t * b0,
1057                                          ip4_header_t * ip0,
1058                                          icmp46_header_t * icmp0,
1059                                          u32 sw_if_index0,
1060                                          u32 rx_fib_index0,
1061                                          vlib_node_runtime_t * node,
1062                                          u32 next0,
1063                                          f64 now,
1064                                          u32 thread_index,
1065                                          snat_session_t ** p_s0)
1066 {
1067   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1068                       next0, thread_index, p_s0, 0);
1069   snat_session_t * s0 = *p_s0;
1070   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1071     {
1072       /* Hairpinning */
1073       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1074         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1075       /* Accounting */
1076       s0->last_heard = now;
1077       s0->total_pkts++;
1078       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1079       /* Per-user LRU list maintenance */
1080       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1081                          s0->per_user_index);
1082       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1083                           s0->per_user_list_head_index,
1084                           s0->per_user_index);
1085     }
1086   return next0;
1087 }
1088 static inline void
1089 snat_hairpinning_unknown_proto (snat_main_t *sm,
1090                                 vlib_buffer_t * b,
1091                                 ip4_header_t * ip)
1092 {
1093   u32 old_addr, new_addr = 0, ti = 0;
1094   clib_bihash_kv_8_8_t kv, value;
1095   clib_bihash_kv_16_8_t s_kv, s_value;
1096   nat_ed_ses_key_t key;
1097   snat_session_key_t m_key;
1098   snat_static_mapping_t *m;
1099   ip_csum_t sum;
1100   snat_session_t *s;
1101
1102   old_addr = ip->dst_address.as_u32;
1103   key.l_addr.as_u32 = ip->dst_address.as_u32;
1104   key.r_addr.as_u32 = ip->src_address.as_u32;
1105   key.fib_index = sm->outside_fib_index;
1106   key.proto = ip->protocol;
1107   key.r_port = 0;
1108   key.l_port = 0;
1109   s_kv.key[0] = key.as_u64[0];
1110   s_kv.key[1] = key.as_u64[1];
1111   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1112     {
1113       m_key.addr = ip->dst_address;
1114       m_key.fib_index = sm->outside_fib_index;
1115       m_key.port = 0;
1116       m_key.protocol = 0;
1117       kv.key = m_key.as_u64;
1118       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1119         return;
1120
1121       m = pool_elt_at_index (sm->static_mappings, value.value);
1122       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1123         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1124       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1125     }
1126   else
1127     {
1128       if (sm->num_workers > 1)
1129         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1130       else
1131         ti = sm->num_workers;
1132
1133       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1134       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1135         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1136       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1137     }
1138   sum = ip->checksum;
1139   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1140   ip->checksum = ip_csum_fold (sum);
1141 }
1142
1143 static snat_session_t *
1144 snat_in2out_unknown_proto (snat_main_t *sm,
1145                            vlib_buffer_t * b,
1146                            ip4_header_t * ip,
1147                            u32 rx_fib_index,
1148                            u32 thread_index,
1149                            f64 now,
1150                            vlib_main_t * vm,
1151                            vlib_node_runtime_t * node)
1152 {
1153   clib_bihash_kv_8_8_t kv, value;
1154   clib_bihash_kv_16_8_t s_kv, s_value;
1155   snat_static_mapping_t *m;
1156   snat_session_key_t m_key;
1157   u32 old_addr, new_addr = 0;
1158   ip_csum_t sum;
1159   snat_user_t *u;
1160   dlist_elt_t *head, *elt;
1161   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1162   u32 elt_index, head_index, ses_index;
1163   snat_session_t * s;
1164   nat_ed_ses_key_t key;
1165   u32 address_index = ~0;
1166   int i;
1167   u8 is_sm = 0;
1168
1169   old_addr = ip->src_address.as_u32;
1170
1171   key.l_addr = ip->src_address;
1172   key.r_addr = ip->dst_address;
1173   key.fib_index = rx_fib_index;
1174   key.proto = ip->protocol;
1175   key.l_port = 0;
1176   key.r_port = 0;
1177   s_kv.key[0] = key.as_u64[0];
1178   s_kv.key[1] = key.as_u64[1];
1179
1180   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1181     {
1182       s = pool_elt_at_index (tsm->sessions, s_value.value);
1183       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1184     }
1185   else
1186     {
1187       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1188         {
1189           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1190           nat_ipfix_logging_max_sessions(sm->max_translations);
1191           return 0;
1192         }
1193
1194       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1195                                   thread_index);
1196       if (!u)
1197         {
1198           clib_warning ("create NAT user failed");
1199           return 0;
1200         }
1201
1202       m_key.addr = ip->src_address;
1203       m_key.port = 0;
1204       m_key.protocol = 0;
1205       m_key.fib_index = rx_fib_index;
1206       kv.key = m_key.as_u64;
1207
1208       /* Try to find static mapping first */
1209       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1210         {
1211           m = pool_elt_at_index (sm->static_mappings, value.value);
1212           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1213           is_sm = 1;
1214           goto create_ses;
1215         }
1216       /* Fallback to 3-tuple key */
1217       else
1218         {
1219           /* Choose same out address as for TCP/UDP session to same destination */
1220           head_index = u->sessions_per_user_list_head_index;
1221           head = pool_elt_at_index (tsm->list_pool, head_index);
1222           elt_index = head->next;
1223           if (PREDICT_FALSE (elt_index == ~0))
1224             ses_index = ~0;
1225           else
1226             {
1227               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1228               ses_index = elt->value;
1229             }
1230
1231           while (ses_index != ~0)
1232             {
1233               s =  pool_elt_at_index (tsm->sessions, ses_index);
1234               elt_index = elt->next;
1235               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1236               ses_index = elt->value;
1237
1238               if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1239                 {
1240                   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1241                   address_index = s->outside_address_index;
1242
1243                   key.fib_index = sm->outside_fib_index;
1244                   key.l_addr.as_u32 = new_addr;
1245                   s_kv.key[0] = key.as_u64[0];
1246                   s_kv.key[1] = key.as_u64[1];
1247                   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1248                     break;
1249
1250                   goto create_ses;
1251                 }
1252             }
1253           key.fib_index = sm->outside_fib_index;
1254           for (i = 0; i < vec_len (sm->addresses); i++)
1255             {
1256               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1257               s_kv.key[0] = key.as_u64[0];
1258               s_kv.key[1] = key.as_u64[1];
1259               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1260                 {
1261                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1262                   address_index = i;
1263                   goto create_ses;
1264                 }
1265             }
1266           return 0;
1267         }
1268
1269 create_ses:
1270       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1271       if (!s)
1272         {
1273           clib_warning ("create NAT session failed");
1274           return 0;
1275         }
1276
1277       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1278       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1279       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1280       s->outside_address_index = address_index;
1281       s->out2in.addr.as_u32 = new_addr;
1282       s->out2in.fib_index = sm->outside_fib_index;
1283       s->in2out.addr.as_u32 = old_addr;
1284       s->in2out.fib_index = rx_fib_index;
1285       s->in2out.port = s->out2in.port = ip->protocol;
1286       if (is_sm)
1287         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1288       user_session_increment (sm, u, is_sm);
1289
1290       /* Add to lookup tables */
1291       key.l_addr.as_u32 = old_addr;
1292       key.r_addr = ip->dst_address;
1293       key.proto = ip->protocol;
1294       key.fib_index = rx_fib_index;
1295       s_kv.key[0] = key.as_u64[0];
1296       s_kv.key[1] = key.as_u64[1];
1297       s_kv.value = s - tsm->sessions;
1298       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1299         clib_warning ("in2out key add failed");
1300
1301       key.l_addr.as_u32 = new_addr;
1302       key.fib_index = sm->outside_fib_index;
1303       s_kv.key[0] = key.as_u64[0];
1304       s_kv.key[1] = key.as_u64[1];
1305       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1306         clib_warning ("out2in key add failed");
1307   }
1308
1309   /* Update IP checksum */
1310   sum = ip->checksum;
1311   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1312   ip->checksum = ip_csum_fold (sum);
1313
1314   /* Accounting */
1315   s->last_heard = now;
1316   s->total_pkts++;
1317   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1318   /* Per-user LRU list maintenance */
1319   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1320   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1321                       s->per_user_index);
1322
1323   /* Hairpinning */
1324   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1325     snat_hairpinning_unknown_proto(sm, b, ip);
1326
1327   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1328     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1329
1330   return s;
1331 }
1332
1333 static snat_session_t *
1334 snat_in2out_lb (snat_main_t *sm,
1335                 vlib_buffer_t * b,
1336                 ip4_header_t * ip,
1337                 u32 rx_fib_index,
1338                 u32 thread_index,
1339                 f64 now,
1340                 vlib_main_t * vm,
1341                 vlib_node_runtime_t * node)
1342 {
1343   nat_ed_ses_key_t key;
1344   clib_bihash_kv_16_8_t s_kv, s_value;
1345   udp_header_t *udp = ip4_next_header (ip);
1346   tcp_header_t *tcp = (tcp_header_t *) udp;
1347   snat_session_t *s = 0;
1348   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1349   u32 old_addr, new_addr;
1350   u16 new_port, old_port;
1351   ip_csum_t sum;
1352   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1353   snat_session_key_t e_key, l_key;
1354   snat_user_t *u;
1355   u8 lb;
1356
1357   old_addr = ip->src_address.as_u32;
1358
1359   key.l_addr = ip->src_address;
1360   key.r_addr = ip->dst_address;
1361   key.fib_index = rx_fib_index;
1362   key.proto = ip->protocol;
1363   key.r_port = udp->dst_port;
1364   key.l_port = udp->src_port;
1365   s_kv.key[0] = key.as_u64[0];
1366   s_kv.key[1] = key.as_u64[1];
1367
1368   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1369     {
1370       s = pool_elt_at_index (tsm->sessions, s_value.value);
1371       if (is_fwd_bypass_session (s))
1372         return 0;
1373     }
1374   else
1375     {
1376       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1377         {
1378           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1379           nat_ipfix_logging_max_sessions(sm->max_translations);
1380           return 0;
1381         }
1382
1383       l_key.addr = ip->src_address;
1384       l_key.port = udp->src_port;
1385       l_key.protocol = proto;
1386       l_key.fib_index = rx_fib_index;
1387       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0, &lb))
1388         return 0;
1389
1390       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1391                                   thread_index);
1392       if (!u)
1393         {
1394           clib_warning ("create NAT user failed");
1395           return 0;
1396         }
1397
1398       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1399       if (!s)
1400         {
1401           clib_warning ("create NAT session failed");
1402           return 0;
1403         }
1404
1405       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1406       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1407       if (lb)
1408         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1409       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1410       s->outside_address_index = ~0;
1411       s->in2out = l_key;
1412       s->out2in = e_key;
1413       s->out2in.protocol = l_key.protocol;
1414       user_session_increment (sm, u, 1 /* static */);
1415
1416       /* Add to lookup tables */
1417       s_kv.value = s - tsm->sessions;
1418       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1419         clib_warning ("in2out-ed key add failed");
1420
1421       key.l_addr = e_key.addr;
1422       key.fib_index = e_key.fib_index;
1423       key.l_port = e_key.port;
1424       s_kv.key[0] = key.as_u64[0];
1425       s_kv.key[1] = key.as_u64[1];
1426       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1427         clib_warning ("out2in-ed key add failed");
1428     }
1429
1430   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1431
1432   /* Update IP checksum */
1433   sum = ip->checksum;
1434   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1435   if (is_twice_nat_session (s))
1436     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1437                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1438   ip->checksum = ip_csum_fold (sum);
1439
1440   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1441     {
1442       old_port = tcp->src_port;
1443       tcp->src_port = s->out2in.port;
1444       new_port = tcp->src_port;
1445
1446       sum = tcp->checksum;
1447       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1448       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1449       if (is_twice_nat_session (s))
1450         {
1451           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1452                                 s->ext_host_addr.as_u32, ip4_header_t,
1453                                 dst_address);
1454           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1455                                 ip4_header_t, length);
1456           tcp->dst_port = s->ext_host_port;
1457           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1458         }
1459       tcp->checksum = ip_csum_fold(sum);
1460     }
1461   else
1462     {
1463       udp->src_port = s->out2in.port;
1464       if (is_twice_nat_session (s))
1465         {
1466           udp->dst_port = s->ext_host_port;
1467           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1468         }
1469       udp->checksum = 0;
1470     }
1471
1472   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1473     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1474
1475   /* Accounting */
1476   s->last_heard = now;
1477   s->total_pkts++;
1478   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1479   /* Per-user LRU list maintenance */
1480   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1481   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1482                       s->per_user_index);
1483   return s;
1484 }
1485
1486 static inline uword
1487 snat_in2out_node_fn_inline (vlib_main_t * vm,
1488                             vlib_node_runtime_t * node,
1489                             vlib_frame_t * frame, int is_slow_path,
1490                             int is_output_feature)
1491 {
1492   u32 n_left_from, * from, * to_next;
1493   snat_in2out_next_t next_index;
1494   u32 pkts_processed = 0;
1495   snat_main_t * sm = &snat_main;
1496   f64 now = vlib_time_now (vm);
1497   u32 stats_node_index;
1498   u32 thread_index = vlib_get_thread_index ();
1499
1500   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1501     snat_in2out_node.index;
1502
1503   from = vlib_frame_vector_args (frame);
1504   n_left_from = frame->n_vectors;
1505   next_index = node->cached_next_index;
1506
1507   while (n_left_from > 0)
1508     {
1509       u32 n_left_to_next;
1510
1511       vlib_get_next_frame (vm, node, next_index,
1512                            to_next, n_left_to_next);
1513
1514       while (n_left_from >= 4 && n_left_to_next >= 2)
1515         {
1516           u32 bi0, bi1;
1517           vlib_buffer_t * b0, * b1;
1518           u32 next0, next1;
1519           u32 sw_if_index0, sw_if_index1;
1520           ip4_header_t * ip0, * ip1;
1521           ip_csum_t sum0, sum1;
1522           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1523           u16 old_port0, new_port0, old_port1, new_port1;
1524           udp_header_t * udp0, * udp1;
1525           tcp_header_t * tcp0, * tcp1;
1526           icmp46_header_t * icmp0, * icmp1;
1527           snat_session_key_t key0, key1;
1528           u32 rx_fib_index0, rx_fib_index1;
1529           u32 proto0, proto1;
1530           snat_session_t * s0 = 0, * s1 = 0;
1531           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1532           u32 iph_offset0 = 0, iph_offset1 = 0;
1533
1534           /* Prefetch next iteration. */
1535           {
1536             vlib_buffer_t * p2, * p3;
1537
1538             p2 = vlib_get_buffer (vm, from[2]);
1539             p3 = vlib_get_buffer (vm, from[3]);
1540
1541             vlib_prefetch_buffer_header (p2, LOAD);
1542             vlib_prefetch_buffer_header (p3, LOAD);
1543
1544             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1545             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1546           }
1547
1548           /* speculatively enqueue b0 and b1 to the current next frame */
1549           to_next[0] = bi0 = from[0];
1550           to_next[1] = bi1 = from[1];
1551           from += 2;
1552           to_next += 2;
1553           n_left_from -= 2;
1554           n_left_to_next -= 2;
1555
1556           b0 = vlib_get_buffer (vm, bi0);
1557           b1 = vlib_get_buffer (vm, bi1);
1558
1559           if (is_output_feature)
1560             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1561
1562           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1563                  iph_offset0);
1564
1565           udp0 = ip4_next_header (ip0);
1566           tcp0 = (tcp_header_t *) udp0;
1567           icmp0 = (icmp46_header_t *) udp0;
1568
1569           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1570           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1571                                    sw_if_index0);
1572
1573           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1574
1575           if (PREDICT_FALSE(ip0->ttl == 1))
1576             {
1577               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1578               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1579                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1580                                            0);
1581               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1582               goto trace00;
1583             }
1584
1585           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1586
1587           /* Next configured feature, probably ip4-lookup */
1588           if (is_slow_path)
1589             {
1590               if (PREDICT_FALSE (proto0 == ~0))
1591                 {
1592                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1593                                                   thread_index, now, vm, node);
1594                   if (!s0)
1595                     next0 = SNAT_IN2OUT_NEXT_DROP;
1596                   goto trace00;
1597                 }
1598
1599               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1600                 {
1601                   next0 = icmp_in2out_slow_path
1602                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1603                      node, next0, now, thread_index, &s0);
1604                   goto trace00;
1605                 }
1606             }
1607           else
1608             {
1609               if (is_output_feature)
1610                 {
1611                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
1612                     goto trace00;
1613                 }
1614
1615               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1616                 {
1617                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1618                   goto trace00;
1619                 }
1620
1621               if (ip4_is_fragment (ip0))
1622                 {
1623                   next0 = SNAT_IN2OUT_NEXT_REASS;
1624                   goto trace00;
1625                 }
1626             }
1627
1628           key0.addr = ip0->src_address;
1629           key0.port = udp0->src_port;
1630           key0.protocol = proto0;
1631           key0.fib_index = rx_fib_index0;
1632
1633           kv0.key = key0.as_u64;
1634
1635           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1636               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1637             {
1638               if (is_slow_path)
1639                 {
1640                   if (is_output_feature)
1641                     {
1642                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1643                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1644                         goto trace00;
1645                     }
1646                   else
1647                     {
1648                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1649                           ip0, proto0, rx_fib_index0, thread_index)))
1650                         goto trace00;
1651                     }
1652
1653                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1654                                      &s0, node, next0, thread_index);
1655                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1656                     goto trace00;
1657                 }
1658               else
1659                 {
1660                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1661                   goto trace00;
1662                 }
1663             }
1664           else
1665             {
1666               if (PREDICT_FALSE (value0.value == ~0ULL))
1667                 {
1668                   if (is_slow_path)
1669                     {
1670                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1671                                           thread_index, now, vm, node);
1672                       if (!s0 && !sm->forwarding_enabled)
1673                         next0 = SNAT_IN2OUT_NEXT_DROP;
1674                       goto trace00;
1675                     }
1676                   else
1677                     {
1678                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1679                       goto trace00;
1680                     }
1681                 }
1682               else
1683                 {
1684                   s0 = pool_elt_at_index (
1685                     sm->per_thread_data[thread_index].sessions,
1686                     value0.value);
1687                 }
1688             }
1689
1690           b0->flags |= VNET_BUFFER_F_IS_NATED;
1691
1692           old_addr0 = ip0->src_address.as_u32;
1693           ip0->src_address = s0->out2in.addr;
1694           new_addr0 = ip0->src_address.as_u32;
1695           if (!is_output_feature)
1696             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1697
1698           sum0 = ip0->checksum;
1699           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1700                                  ip4_header_t,
1701                                  src_address /* changed member */);
1702           ip0->checksum = ip_csum_fold (sum0);
1703
1704           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1705             {
1706               old_port0 = tcp0->src_port;
1707               tcp0->src_port = s0->out2in.port;
1708               new_port0 = tcp0->src_port;
1709
1710               sum0 = tcp0->checksum;
1711               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1712                                      ip4_header_t,
1713                                      dst_address /* changed member */);
1714               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1715                                      ip4_header_t /* cheat */,
1716                                      length /* changed member */);
1717               tcp0->checksum = ip_csum_fold(sum0);
1718             }
1719           else
1720             {
1721               old_port0 = udp0->src_port;
1722               udp0->src_port = s0->out2in.port;
1723               udp0->checksum = 0;
1724             }
1725
1726           /* Accounting */
1727           s0->last_heard = now;
1728           s0->total_pkts++;
1729           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1730           /* Per-user LRU list maintenance */
1731           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1732                              s0->per_user_index);
1733           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1734                               s0->per_user_list_head_index,
1735                               s0->per_user_index);
1736         trace00:
1737
1738           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1739                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1740             {
1741               snat_in2out_trace_t *t =
1742                  vlib_add_trace (vm, node, b0, sizeof (*t));
1743               t->is_slow_path = is_slow_path;
1744               t->sw_if_index = sw_if_index0;
1745               t->next_index = next0;
1746                   t->session_index = ~0;
1747               if (s0)
1748                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1749             }
1750
1751           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1752
1753           if (is_output_feature)
1754             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1755
1756           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1757                  iph_offset1);
1758
1759           udp1 = ip4_next_header (ip1);
1760           tcp1 = (tcp_header_t *) udp1;
1761           icmp1 = (icmp46_header_t *) udp1;
1762
1763           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1764           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1765                                    sw_if_index1);
1766
1767           if (PREDICT_FALSE(ip1->ttl == 1))
1768             {
1769               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1770               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1771                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1772                                            0);
1773               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1774               goto trace01;
1775             }
1776
1777           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1778
1779           /* Next configured feature, probably ip4-lookup */
1780           if (is_slow_path)
1781             {
1782               if (PREDICT_FALSE (proto1 == ~0))
1783                 {
1784                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1785                                                   thread_index, now, vm, node);
1786                   if (!s1)
1787                     next1 = SNAT_IN2OUT_NEXT_DROP;
1788                   goto trace01;
1789                 }
1790
1791               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1792                 {
1793                   next1 = icmp_in2out_slow_path
1794                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1795                      next1, now, thread_index, &s1);
1796                   goto trace01;
1797                 }
1798             }
1799           else
1800             {
1801               if (is_output_feature)
1802                 {
1803                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip1, thread_index)))
1804                     goto trace01;
1805                 }
1806
1807               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1808                 {
1809                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1810                   goto trace01;
1811                 }
1812
1813               if (ip4_is_fragment (ip1))
1814                 {
1815                   next1 = SNAT_IN2OUT_NEXT_REASS;
1816                   goto trace01;
1817                 }
1818             }
1819
1820           key1.addr = ip1->src_address;
1821           key1.port = udp1->src_port;
1822           key1.protocol = proto1;
1823           key1.fib_index = rx_fib_index1;
1824
1825           kv1.key = key1.as_u64;
1826
1827             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1828                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1829             {
1830               if (is_slow_path)
1831                 {
1832                   if (is_output_feature)
1833                     {
1834                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1835                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1836                         goto trace01;
1837                     }
1838                   else
1839                     {
1840                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1841                           ip1, proto1, rx_fib_index1, thread_index)))
1842                         goto trace01;
1843                     }
1844
1845                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1846                                      &s1, node, next1, thread_index);
1847                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1848                     goto trace01;
1849                 }
1850               else
1851                 {
1852                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1853                   goto trace01;
1854                 }
1855             }
1856           else
1857             {
1858               if (PREDICT_FALSE (value1.value == ~0ULL))
1859                 {
1860                   if (is_slow_path)
1861                     {
1862                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1863                                           thread_index, now, vm, node);
1864                       if (!s1 && !sm->forwarding_enabled)
1865                         next1 = SNAT_IN2OUT_NEXT_DROP;
1866                       goto trace01;
1867                     }
1868                   else
1869                     {
1870                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1871                       goto trace01;
1872                     }
1873                 }
1874               else
1875                 {
1876                   s1 = pool_elt_at_index (
1877                     sm->per_thread_data[thread_index].sessions,
1878                     value1.value);
1879                 }
1880             }
1881
1882           b1->flags |= VNET_BUFFER_F_IS_NATED;
1883
1884           old_addr1 = ip1->src_address.as_u32;
1885           ip1->src_address = s1->out2in.addr;
1886           new_addr1 = ip1->src_address.as_u32;
1887           if (!is_output_feature)
1888             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1889
1890           sum1 = ip1->checksum;
1891           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1892                                  ip4_header_t,
1893                                  src_address /* changed member */);
1894           ip1->checksum = ip_csum_fold (sum1);
1895
1896           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1897             {
1898               old_port1 = tcp1->src_port;
1899               tcp1->src_port = s1->out2in.port;
1900               new_port1 = tcp1->src_port;
1901
1902               sum1 = tcp1->checksum;
1903               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1904                                      ip4_header_t,
1905                                      dst_address /* changed member */);
1906               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1907                                      ip4_header_t /* cheat */,
1908                                      length /* changed member */);
1909               tcp1->checksum = ip_csum_fold(sum1);
1910             }
1911           else
1912             {
1913               old_port1 = udp1->src_port;
1914               udp1->src_port = s1->out2in.port;
1915               udp1->checksum = 0;
1916             }
1917
1918           /* Accounting */
1919           s1->last_heard = now;
1920           s1->total_pkts++;
1921           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1922           /* Per-user LRU list maintenance */
1923           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1924                              s1->per_user_index);
1925           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1926                               s1->per_user_list_head_index,
1927                               s1->per_user_index);
1928         trace01:
1929
1930           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1931                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1932             {
1933               snat_in2out_trace_t *t =
1934                  vlib_add_trace (vm, node, b1, sizeof (*t));
1935               t->sw_if_index = sw_if_index1;
1936               t->next_index = next1;
1937               t->session_index = ~0;
1938               if (s1)
1939                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1940             }
1941
1942           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1943
1944           /* verify speculative enqueues, maybe switch current next frame */
1945           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1946                                            to_next, n_left_to_next,
1947                                            bi0, bi1, next0, next1);
1948         }
1949
1950       while (n_left_from > 0 && n_left_to_next > 0)
1951         {
1952           u32 bi0;
1953           vlib_buffer_t * b0;
1954           u32 next0;
1955           u32 sw_if_index0;
1956           ip4_header_t * ip0;
1957           ip_csum_t sum0;
1958           u32 new_addr0, old_addr0;
1959           u16 old_port0, new_port0;
1960           udp_header_t * udp0;
1961           tcp_header_t * tcp0;
1962           icmp46_header_t * icmp0;
1963           snat_session_key_t key0;
1964           u32 rx_fib_index0;
1965           u32 proto0;
1966           snat_session_t * s0 = 0;
1967           clib_bihash_kv_8_8_t kv0, value0;
1968           u32 iph_offset0 = 0;
1969
1970           /* speculatively enqueue b0 to the current next frame */
1971           bi0 = from[0];
1972           to_next[0] = bi0;
1973           from += 1;
1974           to_next += 1;
1975           n_left_from -= 1;
1976           n_left_to_next -= 1;
1977
1978           b0 = vlib_get_buffer (vm, bi0);
1979           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1980
1981           if (is_output_feature)
1982             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1983
1984           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1985                  iph_offset0);
1986
1987           udp0 = ip4_next_header (ip0);
1988           tcp0 = (tcp_header_t *) udp0;
1989           icmp0 = (icmp46_header_t *) udp0;
1990
1991           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1992           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1993                                    sw_if_index0);
1994
1995           if (PREDICT_FALSE(ip0->ttl == 1))
1996             {
1997               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1998               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1999                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2000                                            0);
2001               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2002               goto trace0;
2003             }
2004
2005           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2006
2007           /* Next configured feature, probably ip4-lookup */
2008           if (is_slow_path)
2009             {
2010               if (PREDICT_FALSE (proto0 == ~0))
2011                 {
2012                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
2013                                                   thread_index, now, vm, node);
2014                   if (!s0)
2015                     next0 = SNAT_IN2OUT_NEXT_DROP;
2016                   goto trace0;
2017                 }
2018
2019               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2020                 {
2021                   next0 = icmp_in2out_slow_path
2022                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2023                      next0, now, thread_index, &s0);
2024                   goto trace0;
2025                 }
2026             }
2027           else
2028             {
2029                if (is_output_feature)
2030                 {
2031                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
2032                     goto trace0;
2033                 }
2034
2035               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2036                 {
2037                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2038                   goto trace0;
2039                 }
2040
2041               if (ip4_is_fragment (ip0))
2042                 {
2043                   next0 = SNAT_IN2OUT_NEXT_REASS;
2044                   goto trace0;
2045                 }
2046             }
2047
2048           key0.addr = ip0->src_address;
2049           key0.port = udp0->src_port;
2050           key0.protocol = proto0;
2051           key0.fib_index = rx_fib_index0;
2052
2053           kv0.key = key0.as_u64;
2054
2055           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2056                                       &kv0, &value0))
2057             {
2058               if (is_slow_path)
2059                 {
2060                   if (is_output_feature)
2061                     {
2062                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
2063                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
2064                         goto trace0;
2065                     }
2066                   else
2067                     {
2068                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2069                           ip0, proto0, rx_fib_index0, thread_index)))
2070                         goto trace0;
2071                     }
2072
2073                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2074                                      &s0, node, next0, thread_index);
2075
2076                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2077                     goto trace0;
2078                 }
2079               else
2080                 {
2081                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2082                   goto trace0;
2083                 }
2084             }
2085           else
2086             {
2087               if (PREDICT_FALSE (value0.value == ~0ULL))
2088                 {
2089                   if (is_slow_path)
2090                     {
2091                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2092                                           thread_index, now, vm, node);
2093                       if (!s0 && !sm->forwarding_enabled)
2094                         next0 = SNAT_IN2OUT_NEXT_DROP;
2095                       goto trace0;
2096                     }
2097                   else
2098                     {
2099                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2100                       goto trace0;
2101                     }
2102                 }
2103               else
2104                 {
2105                   s0 = pool_elt_at_index (
2106                     sm->per_thread_data[thread_index].sessions,
2107                     value0.value);
2108                 }
2109             }
2110
2111           b0->flags |= VNET_BUFFER_F_IS_NATED;
2112
2113           old_addr0 = ip0->src_address.as_u32;
2114           ip0->src_address = s0->out2in.addr;
2115           new_addr0 = ip0->src_address.as_u32;
2116           if (!is_output_feature)
2117             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2118
2119           sum0 = ip0->checksum;
2120           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2121                                  ip4_header_t,
2122                                  src_address /* changed member */);
2123           ip0->checksum = ip_csum_fold (sum0);
2124
2125           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2126             {
2127               old_port0 = tcp0->src_port;
2128               tcp0->src_port = s0->out2in.port;
2129               new_port0 = tcp0->src_port;
2130
2131               sum0 = tcp0->checksum;
2132               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2133                                      ip4_header_t,
2134                                      dst_address /* changed member */);
2135               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2136                                      ip4_header_t /* cheat */,
2137                                      length /* changed member */);
2138               tcp0->checksum = ip_csum_fold(sum0);
2139             }
2140           else
2141             {
2142               old_port0 = udp0->src_port;
2143               udp0->src_port = s0->out2in.port;
2144               udp0->checksum = 0;
2145             }
2146
2147           /* Accounting */
2148           s0->last_heard = now;
2149           s0->total_pkts++;
2150           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2151           /* Per-user LRU list maintenance */
2152           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2153                              s0->per_user_index);
2154           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2155                               s0->per_user_list_head_index,
2156                               s0->per_user_index);
2157
2158         trace0:
2159           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2160                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2161             {
2162               snat_in2out_trace_t *t =
2163                  vlib_add_trace (vm, node, b0, sizeof (*t));
2164               t->is_slow_path = is_slow_path;
2165               t->sw_if_index = sw_if_index0;
2166               t->next_index = next0;
2167                   t->session_index = ~0;
2168               if (s0)
2169                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2170             }
2171
2172           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2173
2174           /* verify speculative enqueue, maybe switch current next frame */
2175           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2176                                            to_next, n_left_to_next,
2177                                            bi0, next0);
2178         }
2179
2180       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2181     }
2182
2183   vlib_node_increment_counter (vm, stats_node_index,
2184                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2185                                pkts_processed);
2186   return frame->n_vectors;
2187 }
2188
2189 static uword
2190 snat_in2out_fast_path_fn (vlib_main_t * vm,
2191                           vlib_node_runtime_t * node,
2192                           vlib_frame_t * frame)
2193 {
2194   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2195 }
2196
2197 VLIB_REGISTER_NODE (snat_in2out_node) = {
2198   .function = snat_in2out_fast_path_fn,
2199   .name = "nat44-in2out",
2200   .vector_size = sizeof (u32),
2201   .format_trace = format_snat_in2out_trace,
2202   .type = VLIB_NODE_TYPE_INTERNAL,
2203
2204   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2205   .error_strings = snat_in2out_error_strings,
2206
2207   .runtime_data_bytes = sizeof (snat_runtime_t),
2208
2209   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2210
2211   /* edit / add dispositions here */
2212   .next_nodes = {
2213     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2214     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2215     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2216     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2217     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2218   },
2219 };
2220
2221 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2222
2223 static uword
2224 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2225                                  vlib_node_runtime_t * node,
2226                                  vlib_frame_t * frame)
2227 {
2228   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2229 }
2230
2231 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2232   .function = snat_in2out_output_fast_path_fn,
2233   .name = "nat44-in2out-output",
2234   .vector_size = sizeof (u32),
2235   .format_trace = format_snat_in2out_trace,
2236   .type = VLIB_NODE_TYPE_INTERNAL,
2237
2238   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2239   .error_strings = snat_in2out_error_strings,
2240
2241   .runtime_data_bytes = sizeof (snat_runtime_t),
2242
2243   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2244
2245   /* edit / add dispositions here */
2246   .next_nodes = {
2247     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2248     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2249     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2250     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2251     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2252   },
2253 };
2254
2255 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2256                               snat_in2out_output_fast_path_fn);
2257
2258 static uword
2259 snat_in2out_slow_path_fn (vlib_main_t * vm,
2260                           vlib_node_runtime_t * node,
2261                           vlib_frame_t * frame)
2262 {
2263   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2264 }
2265
2266 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2267   .function = snat_in2out_slow_path_fn,
2268   .name = "nat44-in2out-slowpath",
2269   .vector_size = sizeof (u32),
2270   .format_trace = format_snat_in2out_trace,
2271   .type = VLIB_NODE_TYPE_INTERNAL,
2272
2273   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2274   .error_strings = snat_in2out_error_strings,
2275
2276   .runtime_data_bytes = sizeof (snat_runtime_t),
2277
2278   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2279
2280   /* edit / add dispositions here */
2281   .next_nodes = {
2282     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2283     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2284     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2285     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2286     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2287   },
2288 };
2289
2290 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2291                               snat_in2out_slow_path_fn);
2292
2293 static uword
2294 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2295                                  vlib_node_runtime_t * node,
2296                                  vlib_frame_t * frame)
2297 {
2298   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2299 }
2300
2301 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2302   .function = snat_in2out_output_slow_path_fn,
2303   .name = "nat44-in2out-output-slowpath",
2304   .vector_size = sizeof (u32),
2305   .format_trace = format_snat_in2out_trace,
2306   .type = VLIB_NODE_TYPE_INTERNAL,
2307
2308   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2309   .error_strings = snat_in2out_error_strings,
2310
2311   .runtime_data_bytes = sizeof (snat_runtime_t),
2312
2313   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2314
2315   /* edit / add dispositions here */
2316   .next_nodes = {
2317     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2318     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2319     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2320     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2321     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2322   },
2323 };
2324
2325 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2326                               snat_in2out_output_slow_path_fn);
2327
2328 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2329
2330 static uword
2331 nat44_hairpinning_fn (vlib_main_t * vm,
2332                       vlib_node_runtime_t * node,
2333                       vlib_frame_t * frame)
2334 {
2335   u32 n_left_from, * from, * to_next;
2336   snat_in2out_next_t next_index;
2337   u32 pkts_processed = 0;
2338   snat_main_t * sm = &snat_main;
2339   vnet_feature_main_t *fm = &feature_main;
2340   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2341   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2342
2343   from = vlib_frame_vector_args (frame);
2344   n_left_from = frame->n_vectors;
2345   next_index = node->cached_next_index;
2346
2347   while (n_left_from > 0)
2348     {
2349       u32 n_left_to_next;
2350
2351       vlib_get_next_frame (vm, node, next_index,
2352                            to_next, n_left_to_next);
2353
2354       while (n_left_from > 0 && n_left_to_next > 0)
2355         {
2356           u32 bi0;
2357           vlib_buffer_t * b0;
2358           u32 next0;
2359           ip4_header_t * ip0;
2360           u32 proto0;
2361           udp_header_t * udp0;
2362           tcp_header_t * tcp0;
2363
2364           /* speculatively enqueue b0 to the current next frame */
2365           bi0 = from[0];
2366           to_next[0] = bi0;
2367           from += 1;
2368           to_next += 1;
2369           n_left_from -= 1;
2370           n_left_to_next -= 1;
2371
2372           b0 = vlib_get_buffer (vm, bi0);
2373           ip0 = vlib_buffer_get_current (b0);
2374           udp0 = ip4_next_header (ip0);
2375           tcp0 = (tcp_header_t *) udp0;
2376
2377           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2378
2379           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2380                                 &next0, 0);
2381
2382           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2383             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2384
2385           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2386
2387           /* verify speculative enqueue, maybe switch current next frame */
2388           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2389                                            to_next, n_left_to_next,
2390                                            bi0, next0);
2391          }
2392
2393       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2394     }
2395
2396   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2397                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2398                                pkts_processed);
2399   return frame->n_vectors;
2400 }
2401
2402 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2403   .function = nat44_hairpinning_fn,
2404   .name = "nat44-hairpinning",
2405   .vector_size = sizeof (u32),
2406   .type = VLIB_NODE_TYPE_INTERNAL,
2407   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2408   .error_strings = snat_in2out_error_strings,
2409   .n_next_nodes = 2,
2410   .next_nodes = {
2411     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2412     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2413   },
2414 };
2415
2416 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2417                               nat44_hairpinning_fn);
2418
2419 static inline void
2420 nat44_reass_hairpinning (snat_main_t *sm,
2421                          vlib_buffer_t * b0,
2422                          ip4_header_t * ip0,
2423                          u16 sport,
2424                          u16 dport,
2425                          u32 proto0)
2426 {
2427   snat_session_key_t key0, sm0;
2428   snat_session_t * s0;
2429   clib_bihash_kv_8_8_t kv0, value0;
2430   ip_csum_t sum0;
2431   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2432   u16 new_dst_port0, old_dst_port0;
2433   udp_header_t * udp0;
2434   tcp_header_t * tcp0;
2435
2436   key0.addr = ip0->dst_address;
2437   key0.port = dport;
2438   key0.protocol = proto0;
2439   key0.fib_index = sm->outside_fib_index;
2440   kv0.key = key0.as_u64;
2441
2442   udp0 = ip4_next_header (ip0);
2443
2444   /* Check if destination is static mappings */
2445   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2446     {
2447       new_dst_addr0 = sm0.addr.as_u32;
2448       new_dst_port0 = sm0.port;
2449       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2450     }
2451   /* or active sessions */
2452   else
2453     {
2454       if (sm->num_workers > 1)
2455         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2456       else
2457         ti = sm->num_workers;
2458
2459       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2460         {
2461           si = value0.value;
2462           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2463           new_dst_addr0 = s0->in2out.addr.as_u32;
2464           new_dst_port0 = s0->in2out.port;
2465           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2466         }
2467     }
2468
2469   /* Destination is behind the same NAT, use internal address and port */
2470   if (new_dst_addr0)
2471     {
2472       old_dst_addr0 = ip0->dst_address.as_u32;
2473       ip0->dst_address.as_u32 = new_dst_addr0;
2474       sum0 = ip0->checksum;
2475       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2476                              ip4_header_t, dst_address);
2477       ip0->checksum = ip_csum_fold (sum0);
2478
2479       old_dst_port0 = dport;
2480       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2481                        ip4_is_first_fragment (ip0)))
2482         {
2483           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2484             {
2485               tcp0 = ip4_next_header (ip0);
2486               tcp0->dst = new_dst_port0;
2487               sum0 = tcp0->checksum;
2488               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2489                                      ip4_header_t, dst_address);
2490               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2491                                      ip4_header_t /* cheat */, length);
2492               tcp0->checksum = ip_csum_fold(sum0);
2493             }
2494           else
2495             {
2496               udp0->dst_port = new_dst_port0;
2497               udp0->checksum = 0;
2498             }
2499         }
2500       else
2501         {
2502           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2503             {
2504               tcp0 = ip4_next_header (ip0);
2505               sum0 = tcp0->checksum;
2506               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2507                                      ip4_header_t, dst_address);
2508               tcp0->checksum = ip_csum_fold(sum0);
2509             }
2510         }
2511     }
2512 }
2513
2514 static uword
2515 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2516                             vlib_node_runtime_t * node,
2517                             vlib_frame_t * frame)
2518 {
2519   u32 n_left_from, *from, *to_next;
2520   snat_in2out_next_t next_index;
2521   u32 pkts_processed = 0;
2522   snat_main_t *sm = &snat_main;
2523   f64 now = vlib_time_now (vm);
2524   u32 thread_index = vlib_get_thread_index ();
2525   snat_main_per_thread_data_t *per_thread_data =
2526     &sm->per_thread_data[thread_index];
2527   u32 *fragments_to_drop = 0;
2528   u32 *fragments_to_loopback = 0;
2529
2530   from = vlib_frame_vector_args (frame);
2531   n_left_from = frame->n_vectors;
2532   next_index = node->cached_next_index;
2533
2534   while (n_left_from > 0)
2535     {
2536       u32 n_left_to_next;
2537
2538       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2539
2540       while (n_left_from > 0 && n_left_to_next > 0)
2541        {
2542           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2543           vlib_buffer_t *b0;
2544           u32 next0;
2545           u8 cached0 = 0;
2546           ip4_header_t *ip0;
2547           nat_reass_ip4_t *reass0;
2548           udp_header_t * udp0;
2549           tcp_header_t * tcp0;
2550           snat_session_key_t key0;
2551           clib_bihash_kv_8_8_t kv0, value0;
2552           snat_session_t * s0 = 0;
2553           u16 old_port0, new_port0;
2554           ip_csum_t sum0;
2555
2556           /* speculatively enqueue b0 to the current next frame */
2557           bi0 = from[0];
2558           to_next[0] = bi0;
2559           from += 1;
2560           to_next += 1;
2561           n_left_from -= 1;
2562           n_left_to_next -= 1;
2563
2564           b0 = vlib_get_buffer (vm, bi0);
2565           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2566
2567           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2568           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2569                                                                sw_if_index0);
2570
2571           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2572             {
2573               next0 = SNAT_IN2OUT_NEXT_DROP;
2574               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2575               goto trace0;
2576             }
2577
2578           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2579           udp0 = ip4_next_header (ip0);
2580           tcp0 = (tcp_header_t *) udp0;
2581           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2582
2583           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2584                                                  ip0->dst_address,
2585                                                  ip0->fragment_id,
2586                                                  ip0->protocol,
2587                                                  1,
2588                                                  &fragments_to_drop);
2589
2590           if (PREDICT_FALSE (!reass0))
2591             {
2592               next0 = SNAT_IN2OUT_NEXT_DROP;
2593               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2594               goto trace0;
2595             }
2596
2597           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2598             {
2599               key0.addr = ip0->src_address;
2600               key0.port = udp0->src_port;
2601               key0.protocol = proto0;
2602               key0.fib_index = rx_fib_index0;
2603               kv0.key = key0.as_u64;
2604
2605               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2606                 {
2607                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2608                       ip0, proto0, rx_fib_index0, thread_index)))
2609                     goto trace0;
2610
2611                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2612                                      &s0, node, next0, thread_index);
2613
2614                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2615                     goto trace0;
2616
2617                   reass0->sess_index = s0 - per_thread_data->sessions;
2618                 }
2619               else
2620                 {
2621                   s0 = pool_elt_at_index (per_thread_data->sessions,
2622                                           value0.value);
2623                   reass0->sess_index = value0.value;
2624                 }
2625               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2626             }
2627           else
2628             {
2629               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2630                 {
2631                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2632                     {
2633                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2634                       next0 = SNAT_IN2OUT_NEXT_DROP;
2635                       goto trace0;
2636                     }
2637                   cached0 = 1;
2638                   goto trace0;
2639                 }
2640               s0 = pool_elt_at_index (per_thread_data->sessions,
2641                                       reass0->sess_index);
2642             }
2643
2644           old_addr0 = ip0->src_address.as_u32;
2645           ip0->src_address = s0->out2in.addr;
2646           new_addr0 = ip0->src_address.as_u32;
2647           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2648
2649           sum0 = ip0->checksum;
2650           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2651                                  ip4_header_t,
2652                                  src_address /* changed member */);
2653           ip0->checksum = ip_csum_fold (sum0);
2654
2655           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2656             {
2657               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2658                 {
2659                   old_port0 = tcp0->src_port;
2660                   tcp0->src_port = s0->out2in.port;
2661                   new_port0 = tcp0->src_port;
2662
2663                   sum0 = tcp0->checksum;
2664                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2665                                          ip4_header_t,
2666                                          dst_address /* changed member */);
2667                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2668                                          ip4_header_t /* cheat */,
2669                                          length /* changed member */);
2670                   tcp0->checksum = ip_csum_fold(sum0);
2671                 }
2672               else
2673                 {
2674                   old_port0 = udp0->src_port;
2675                   udp0->src_port = s0->out2in.port;
2676                   udp0->checksum = 0;
2677                 }
2678             }
2679
2680           /* Hairpinning */
2681           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2682                                    s0->ext_host_port, proto0);
2683
2684           /* Accounting */
2685           s0->last_heard = now;
2686           s0->total_pkts++;
2687           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2688           /* Per-user LRU list maintenance */
2689           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2690                              s0->per_user_index);
2691           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2692                               s0->per_user_list_head_index,
2693                               s0->per_user_index);
2694
2695         trace0:
2696           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2697                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2698             {
2699               nat44_in2out_reass_trace_t *t =
2700                  vlib_add_trace (vm, node, b0, sizeof (*t));
2701               t->cached = cached0;
2702               t->sw_if_index = sw_if_index0;
2703               t->next_index = next0;
2704             }
2705
2706           if (cached0)
2707             {
2708               n_left_to_next++;
2709               to_next--;
2710             }
2711           else
2712             {
2713               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2714
2715               /* verify speculative enqueue, maybe switch current next frame */
2716               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2717                                                to_next, n_left_to_next,
2718                                                bi0, next0);
2719             }
2720
2721           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2722             {
2723               from = vlib_frame_vector_args (frame);
2724               u32 len = vec_len (fragments_to_loopback);
2725               if (len <= VLIB_FRAME_SIZE)
2726                 {
2727                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2728                   n_left_from = len;
2729                   vec_reset_length (fragments_to_loopback);
2730                 }
2731               else
2732                 {
2733                   clib_memcpy (from,
2734                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2735                                sizeof (u32) * VLIB_FRAME_SIZE);
2736                   n_left_from = VLIB_FRAME_SIZE;
2737                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2738                 }
2739             }
2740        }
2741
2742       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2743     }
2744
2745   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2746                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2747                                pkts_processed);
2748
2749   nat_send_all_to_node (vm, fragments_to_drop, node,
2750                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2751                         SNAT_IN2OUT_NEXT_DROP);
2752
2753   vec_free (fragments_to_drop);
2754   vec_free (fragments_to_loopback);
2755   return frame->n_vectors;
2756 }
2757
2758 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2759   .function = nat44_in2out_reass_node_fn,
2760   .name = "nat44-in2out-reass",
2761   .vector_size = sizeof (u32),
2762   .format_trace = format_nat44_in2out_reass_trace,
2763   .type = VLIB_NODE_TYPE_INTERNAL,
2764
2765   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2766   .error_strings = snat_in2out_error_strings,
2767
2768   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2769   .next_nodes = {
2770     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2771     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2772     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2773     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2774     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2775   },
2776 };
2777
2778 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2779                               nat44_in2out_reass_node_fn);
2780
2781 /**************************/
2782 /*** deterministic mode ***/
2783 /**************************/
2784 static uword
2785 snat_det_in2out_node_fn (vlib_main_t * vm,
2786                          vlib_node_runtime_t * node,
2787                          vlib_frame_t * frame)
2788 {
2789   u32 n_left_from, * from, * to_next;
2790   snat_in2out_next_t next_index;
2791   u32 pkts_processed = 0;
2792   snat_main_t * sm = &snat_main;
2793   u32 now = (u32) vlib_time_now (vm);
2794   u32 thread_index = vlib_get_thread_index ();
2795
2796   from = vlib_frame_vector_args (frame);
2797   n_left_from = frame->n_vectors;
2798   next_index = node->cached_next_index;
2799
2800   while (n_left_from > 0)
2801     {
2802       u32 n_left_to_next;
2803
2804       vlib_get_next_frame (vm, node, next_index,
2805                            to_next, n_left_to_next);
2806
2807       while (n_left_from >= 4 && n_left_to_next >= 2)
2808         {
2809           u32 bi0, bi1;
2810           vlib_buffer_t * b0, * b1;
2811           u32 next0, next1;
2812           u32 sw_if_index0, sw_if_index1;
2813           ip4_header_t * ip0, * ip1;
2814           ip_csum_t sum0, sum1;
2815           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2816           u16 old_port0, new_port0, lo_port0, i0;
2817           u16 old_port1, new_port1, lo_port1, i1;
2818           udp_header_t * udp0, * udp1;
2819           tcp_header_t * tcp0, * tcp1;
2820           u32 proto0, proto1;
2821           snat_det_out_key_t key0, key1;
2822           snat_det_map_t * dm0, * dm1;
2823           snat_det_session_t * ses0 = 0, * ses1 = 0;
2824           u32 rx_fib_index0, rx_fib_index1;
2825           icmp46_header_t * icmp0, * icmp1;
2826
2827           /* Prefetch next iteration. */
2828           {
2829             vlib_buffer_t * p2, * p3;
2830
2831             p2 = vlib_get_buffer (vm, from[2]);
2832             p3 = vlib_get_buffer (vm, from[3]);
2833
2834             vlib_prefetch_buffer_header (p2, LOAD);
2835             vlib_prefetch_buffer_header (p3, LOAD);
2836
2837             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2838             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2839           }
2840
2841           /* speculatively enqueue b0 and b1 to the current next frame */
2842           to_next[0] = bi0 = from[0];
2843           to_next[1] = bi1 = from[1];
2844           from += 2;
2845           to_next += 2;
2846           n_left_from -= 2;
2847           n_left_to_next -= 2;
2848
2849           b0 = vlib_get_buffer (vm, bi0);
2850           b1 = vlib_get_buffer (vm, bi1);
2851
2852           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2853           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2854
2855           ip0 = vlib_buffer_get_current (b0);
2856           udp0 = ip4_next_header (ip0);
2857           tcp0 = (tcp_header_t *) udp0;
2858
2859           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2860
2861           if (PREDICT_FALSE(ip0->ttl == 1))
2862             {
2863               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2864               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2865                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2866                                            0);
2867               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2868               goto trace0;
2869             }
2870
2871           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2872
2873           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2874             {
2875               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2876               icmp0 = (icmp46_header_t *) udp0;
2877
2878               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2879                                   rx_fib_index0, node, next0, thread_index,
2880                                   &ses0, &dm0);
2881               goto trace0;
2882             }
2883
2884           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2885           if (PREDICT_FALSE(!dm0))
2886             {
2887               clib_warning("no match for internal host %U",
2888                            format_ip4_address, &ip0->src_address);
2889               next0 = SNAT_IN2OUT_NEXT_DROP;
2890               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2891               goto trace0;
2892             }
2893
2894           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2895
2896           key0.ext_host_addr = ip0->dst_address;
2897           key0.ext_host_port = tcp0->dst;
2898
2899           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2900           if (PREDICT_FALSE(!ses0))
2901             {
2902               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2903                 {
2904                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2905                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2906
2907                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2908                     continue;
2909
2910                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2911                   break;
2912                 }
2913               if (PREDICT_FALSE(!ses0))
2914                 {
2915                   /* too many sessions for user, send ICMP error packet */
2916
2917                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2918                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2919                                                ICMP4_destination_unreachable_destination_unreachable_host,
2920                                                0);
2921                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2922                   goto trace0;
2923                 }
2924             }
2925
2926           new_port0 = ses0->out.out_port;
2927
2928           old_addr0.as_u32 = ip0->src_address.as_u32;
2929           ip0->src_address.as_u32 = new_addr0.as_u32;
2930           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2931
2932           sum0 = ip0->checksum;
2933           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2934                                  ip4_header_t,
2935                                  src_address /* changed member */);
2936           ip0->checksum = ip_csum_fold (sum0);
2937
2938           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2939             {
2940               if (tcp0->flags & TCP_FLAG_SYN)
2941                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2942               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2943                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2944               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2945                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2946               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2947                 snat_det_ses_close(dm0, ses0);
2948               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2949                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2950               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2951                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2952
2953               old_port0 = tcp0->src;
2954               tcp0->src = new_port0;
2955
2956               sum0 = tcp0->checksum;
2957               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2958                                      ip4_header_t,
2959                                      dst_address /* changed member */);
2960               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2961                                      ip4_header_t /* cheat */,
2962                                      length /* changed member */);
2963               tcp0->checksum = ip_csum_fold(sum0);
2964             }
2965           else
2966             {
2967               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2968               old_port0 = udp0->src_port;
2969               udp0->src_port = new_port0;
2970               udp0->checksum = 0;
2971             }
2972
2973           switch(ses0->state)
2974             {
2975             case SNAT_SESSION_UDP_ACTIVE:
2976                 ses0->expire = now + sm->udp_timeout;
2977                 break;
2978             case SNAT_SESSION_TCP_SYN_SENT:
2979             case SNAT_SESSION_TCP_FIN_WAIT:
2980             case SNAT_SESSION_TCP_CLOSE_WAIT:
2981             case SNAT_SESSION_TCP_LAST_ACK:
2982                 ses0->expire = now + sm->tcp_transitory_timeout;
2983                 break;
2984             case SNAT_SESSION_TCP_ESTABLISHED:
2985                 ses0->expire = now + sm->tcp_established_timeout;
2986                 break;
2987             }
2988
2989         trace0:
2990           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2991                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2992             {
2993               snat_in2out_trace_t *t =
2994                  vlib_add_trace (vm, node, b0, sizeof (*t));
2995               t->is_slow_path = 0;
2996               t->sw_if_index = sw_if_index0;
2997               t->next_index = next0;
2998               t->session_index = ~0;
2999               if (ses0)
3000                 t->session_index = ses0 - dm0->sessions;
3001             }
3002
3003           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3004
3005           ip1 = vlib_buffer_get_current (b1);
3006           udp1 = ip4_next_header (ip1);
3007           tcp1 = (tcp_header_t *) udp1;
3008
3009           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3010
3011           if (PREDICT_FALSE(ip1->ttl == 1))
3012             {
3013               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3014               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3015                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3016                                            0);
3017               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3018               goto trace1;
3019             }
3020
3021           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3022
3023           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3024             {
3025               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3026               icmp1 = (icmp46_header_t *) udp1;
3027
3028               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
3029                                   rx_fib_index1, node, next1, thread_index,
3030                                   &ses1, &dm1);
3031               goto trace1;
3032             }
3033
3034           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
3035           if (PREDICT_FALSE(!dm1))
3036             {
3037               clib_warning("no match for internal host %U",
3038                            format_ip4_address, &ip0->src_address);
3039               next1 = SNAT_IN2OUT_NEXT_DROP;
3040               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3041               goto trace1;
3042             }
3043
3044           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3045
3046           key1.ext_host_addr = ip1->dst_address;
3047           key1.ext_host_port = tcp1->dst;
3048
3049           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3050           if (PREDICT_FALSE(!ses1))
3051             {
3052               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3053                 {
3054                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
3055                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3056
3057                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3058                     continue;
3059
3060                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3061                   break;
3062                 }
3063               if (PREDICT_FALSE(!ses1))
3064                 {
3065                   /* too many sessions for user, send ICMP error packet */
3066
3067                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3068                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3069                                                ICMP4_destination_unreachable_destination_unreachable_host,
3070                                                0);
3071                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3072                   goto trace1;
3073                 }
3074             }
3075
3076           new_port1 = ses1->out.out_port;
3077
3078           old_addr1.as_u32 = ip1->src_address.as_u32;
3079           ip1->src_address.as_u32 = new_addr1.as_u32;
3080           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3081
3082           sum1 = ip1->checksum;
3083           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3084                                  ip4_header_t,
3085                                  src_address /* changed member */);
3086           ip1->checksum = ip_csum_fold (sum1);
3087
3088           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3089             {
3090               if (tcp1->flags & TCP_FLAG_SYN)
3091                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3092               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3093                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3094               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3095                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3096               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3097                 snat_det_ses_close(dm1, ses1);
3098               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3099                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3100               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3101                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3102
3103               old_port1 = tcp1->src;
3104               tcp1->src = new_port1;
3105
3106               sum1 = tcp1->checksum;
3107               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3108                                      ip4_header_t,
3109                                      dst_address /* changed member */);
3110               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3111                                      ip4_header_t /* cheat */,
3112                                      length /* changed member */);
3113               tcp1->checksum = ip_csum_fold(sum1);
3114             }
3115           else
3116             {
3117               ses1->state = SNAT_SESSION_UDP_ACTIVE;
3118               old_port1 = udp1->src_port;
3119               udp1->src_port = new_port1;
3120               udp1->checksum = 0;
3121             }
3122
3123           switch(ses1->state)
3124             {
3125             case SNAT_SESSION_UDP_ACTIVE:
3126                 ses1->expire = now + sm->udp_timeout;
3127                 break;
3128             case SNAT_SESSION_TCP_SYN_SENT:
3129             case SNAT_SESSION_TCP_FIN_WAIT:
3130             case SNAT_SESSION_TCP_CLOSE_WAIT:
3131             case SNAT_SESSION_TCP_LAST_ACK:
3132                 ses1->expire = now + sm->tcp_transitory_timeout;
3133                 break;
3134             case SNAT_SESSION_TCP_ESTABLISHED:
3135                 ses1->expire = now + sm->tcp_established_timeout;
3136                 break;
3137             }
3138
3139         trace1:
3140           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3141                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3142             {
3143               snat_in2out_trace_t *t =
3144                  vlib_add_trace (vm, node, b1, sizeof (*t));
3145               t->is_slow_path = 0;
3146               t->sw_if_index = sw_if_index1;
3147               t->next_index = next1;
3148               t->session_index = ~0;
3149               if (ses1)
3150                 t->session_index = ses1 - dm1->sessions;
3151             }
3152
3153           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3154
3155           /* verify speculative enqueues, maybe switch current next frame */
3156           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3157                                            to_next, n_left_to_next,
3158                                            bi0, bi1, next0, next1);
3159          }
3160
3161       while (n_left_from > 0 && n_left_to_next > 0)
3162         {
3163           u32 bi0;
3164           vlib_buffer_t * b0;
3165           u32 next0;
3166           u32 sw_if_index0;
3167           ip4_header_t * ip0;
3168           ip_csum_t sum0;
3169           ip4_address_t new_addr0, old_addr0;
3170           u16 old_port0, new_port0, lo_port0, i0;
3171           udp_header_t * udp0;
3172           tcp_header_t * tcp0;
3173           u32 proto0;
3174           snat_det_out_key_t key0;
3175           snat_det_map_t * dm0;
3176           snat_det_session_t * ses0 = 0;
3177           u32 rx_fib_index0;
3178           icmp46_header_t * icmp0;
3179
3180           /* speculatively enqueue b0 to the current next frame */
3181           bi0 = from[0];
3182           to_next[0] = bi0;
3183           from += 1;
3184           to_next += 1;
3185           n_left_from -= 1;
3186           n_left_to_next -= 1;
3187
3188           b0 = vlib_get_buffer (vm, bi0);
3189           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3190
3191           ip0 = vlib_buffer_get_current (b0);
3192           udp0 = ip4_next_header (ip0);
3193           tcp0 = (tcp_header_t *) udp0;
3194
3195           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3196
3197           if (PREDICT_FALSE(ip0->ttl == 1))
3198             {
3199               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3200               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3201                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3202                                            0);
3203               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3204               goto trace00;
3205             }
3206
3207           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3208
3209           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3210             {
3211               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3212               icmp0 = (icmp46_header_t *) udp0;
3213
3214               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3215                                   rx_fib_index0, node, next0, thread_index,
3216                                   &ses0, &dm0);
3217               goto trace00;
3218             }
3219
3220           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3221           if (PREDICT_FALSE(!dm0))
3222             {
3223               clib_warning("no match for internal host %U",
3224                            format_ip4_address, &ip0->src_address);
3225               next0 = SNAT_IN2OUT_NEXT_DROP;
3226               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3227               goto trace00;
3228             }
3229
3230           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3231
3232           key0.ext_host_addr = ip0->dst_address;
3233           key0.ext_host_port = tcp0->dst;
3234
3235           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3236           if (PREDICT_FALSE(!ses0))
3237             {
3238               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3239                 {
3240                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3241                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3242
3243                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3244                     continue;
3245
3246                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3247                   break;
3248                 }
3249               if (PREDICT_FALSE(!ses0))
3250                 {
3251                   /* too many sessions for user, send ICMP error packet */
3252
3253                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3254                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3255                                                ICMP4_destination_unreachable_destination_unreachable_host,
3256                                                0);
3257                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3258                   goto trace00;
3259                 }
3260             }
3261
3262           new_port0 = ses0->out.out_port;
3263
3264           old_addr0.as_u32 = ip0->src_address.as_u32;
3265           ip0->src_address.as_u32 = new_addr0.as_u32;
3266           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3267
3268           sum0 = ip0->checksum;
3269           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3270                                  ip4_header_t,
3271                                  src_address /* changed member */);
3272           ip0->checksum = ip_csum_fold (sum0);
3273
3274           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3275             {
3276               if (tcp0->flags & TCP_FLAG_SYN)
3277                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3278               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3279                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3280               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3281                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3282               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3283                 snat_det_ses_close(dm0, ses0);
3284               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3285                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3286               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3287                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3288
3289               old_port0 = tcp0->src;
3290               tcp0->src = new_port0;
3291
3292               sum0 = tcp0->checksum;
3293               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3294                                      ip4_header_t,
3295                                      dst_address /* changed member */);
3296               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3297                                      ip4_header_t /* cheat */,
3298                                      length /* changed member */);
3299               tcp0->checksum = ip_csum_fold(sum0);
3300             }
3301           else
3302             {
3303               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3304               old_port0 = udp0->src_port;
3305               udp0->src_port = new_port0;
3306               udp0->checksum = 0;
3307             }
3308
3309           switch(ses0->state)
3310             {
3311             case SNAT_SESSION_UDP_ACTIVE:
3312                 ses0->expire = now + sm->udp_timeout;
3313                 break;
3314             case SNAT_SESSION_TCP_SYN_SENT:
3315             case SNAT_SESSION_TCP_FIN_WAIT:
3316             case SNAT_SESSION_TCP_CLOSE_WAIT:
3317             case SNAT_SESSION_TCP_LAST_ACK:
3318                 ses0->expire = now + sm->tcp_transitory_timeout;
3319                 break;
3320             case SNAT_SESSION_TCP_ESTABLISHED:
3321                 ses0->expire = now + sm->tcp_established_timeout;
3322                 break;
3323             }
3324
3325         trace00:
3326           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3327                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3328             {
3329               snat_in2out_trace_t *t =
3330                  vlib_add_trace (vm, node, b0, sizeof (*t));
3331               t->is_slow_path = 0;
3332               t->sw_if_index = sw_if_index0;
3333               t->next_index = next0;
3334               t->session_index = ~0;
3335               if (ses0)
3336                 t->session_index = ses0 - dm0->sessions;
3337             }
3338
3339           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3340
3341           /* verify speculative enqueue, maybe switch current next frame */
3342           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3343                                            to_next, n_left_to_next,
3344                                            bi0, next0);
3345         }
3346
3347       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3348     }
3349
3350   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3351                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3352                                pkts_processed);
3353   return frame->n_vectors;
3354 }
3355
3356 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3357   .function = snat_det_in2out_node_fn,
3358   .name = "nat44-det-in2out",
3359   .vector_size = sizeof (u32),
3360   .format_trace = format_snat_in2out_trace,
3361   .type = VLIB_NODE_TYPE_INTERNAL,
3362
3363   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3364   .error_strings = snat_in2out_error_strings,
3365
3366   .runtime_data_bytes = sizeof (snat_runtime_t),
3367
3368   .n_next_nodes = 3,
3369
3370   /* edit / add dispositions here */
3371   .next_nodes = {
3372     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3373     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3374     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3375   },
3376 };
3377
3378 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3379
3380 /**
3381  * Get address and port values to be used for ICMP packet translation
3382  * and create session if needed
3383  *
3384  * @param[in,out] sm             NAT main
3385  * @param[in,out] node           NAT node runtime
3386  * @param[in] thread_index       thread index
3387  * @param[in,out] b0             buffer containing packet to be translated
3388  * @param[out] p_proto           protocol used for matching
3389  * @param[out] p_value           address and port after NAT translation
3390  * @param[out] p_dont_translate  if packet should not be translated
3391  * @param d                      optional parameter
3392  * @param e                      optional parameter
3393  */
3394 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3395                           u32 thread_index, vlib_buffer_t *b0,
3396                           ip4_header_t *ip0, u8 *p_proto,
3397                           snat_session_key_t *p_value,
3398                           u8 *p_dont_translate, void *d, void *e)
3399 {
3400   icmp46_header_t *icmp0;
3401   u32 sw_if_index0;
3402   u32 rx_fib_index0;
3403   u8 protocol;
3404   snat_det_out_key_t key0;
3405   u8 dont_translate = 0;
3406   u32 next0 = ~0;
3407   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3408   ip4_header_t *inner_ip0;
3409   void *l4_header = 0;
3410   icmp46_header_t *inner_icmp0;
3411   snat_det_map_t * dm0 = 0;
3412   ip4_address_t new_addr0;
3413   u16 lo_port0, i0;
3414   snat_det_session_t * ses0 = 0;
3415   ip4_address_t in_addr;
3416   u16 in_port;
3417
3418   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3419   echo0 = (icmp_echo_header_t *)(icmp0+1);
3420   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3421   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3422
3423   if (!icmp_is_error_message (icmp0))
3424     {
3425       protocol = SNAT_PROTOCOL_ICMP;
3426       in_addr = ip0->src_address;
3427       in_port = echo0->identifier;
3428     }
3429   else
3430     {
3431       inner_ip0 = (ip4_header_t *)(echo0+1);
3432       l4_header = ip4_next_header (inner_ip0);
3433       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3434       in_addr = inner_ip0->dst_address;
3435       switch (protocol)
3436         {
3437         case SNAT_PROTOCOL_ICMP:
3438           inner_icmp0 = (icmp46_header_t*)l4_header;
3439           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3440           in_port = inner_echo0->identifier;
3441           break;
3442         case SNAT_PROTOCOL_UDP:
3443         case SNAT_PROTOCOL_TCP:
3444           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3445           break;
3446         default:
3447           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3448           next0 = SNAT_IN2OUT_NEXT_DROP;
3449           goto out;
3450         }
3451     }
3452
3453   dm0 = snat_det_map_by_user(sm, &in_addr);
3454   if (PREDICT_FALSE(!dm0))
3455     {
3456       clib_warning("no match for internal host %U",
3457                    format_ip4_address, &in_addr);
3458       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3459           IP_PROTOCOL_ICMP, rx_fib_index0)))
3460         {
3461           dont_translate = 1;
3462           goto out;
3463         }
3464       next0 = SNAT_IN2OUT_NEXT_DROP;
3465       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3466       goto out;
3467     }
3468
3469   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3470
3471   key0.ext_host_addr = ip0->dst_address;
3472   key0.ext_host_port = 0;
3473
3474   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3475   if (PREDICT_FALSE(!ses0))
3476     {
3477       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3478           IP_PROTOCOL_ICMP, rx_fib_index0)))
3479         {
3480           dont_translate = 1;
3481           goto out;
3482         }
3483       if (icmp0->type != ICMP4_echo_request)
3484         {
3485           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3486           next0 = SNAT_IN2OUT_NEXT_DROP;
3487           goto out;
3488         }
3489       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3490         {
3491           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3492             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3493
3494           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3495             continue;
3496
3497           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3498           break;
3499         }
3500       if (PREDICT_FALSE(!ses0))
3501         {
3502           next0 = SNAT_IN2OUT_NEXT_DROP;
3503           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3504           goto out;
3505         }
3506     }
3507
3508   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3509                     !icmp_is_error_message (icmp0)))
3510     {
3511       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3512       next0 = SNAT_IN2OUT_NEXT_DROP;
3513       goto out;
3514     }
3515
3516   u32 now = (u32) vlib_time_now (sm->vlib_main);
3517
3518   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3519   ses0->expire = now + sm->icmp_timeout;
3520
3521 out:
3522   *p_proto = protocol;
3523   if (ses0)
3524     {
3525       p_value->addr = new_addr0;
3526       p_value->fib_index = sm->outside_fib_index;
3527       p_value->port = ses0->out.out_port;
3528     }
3529   *p_dont_translate = dont_translate;
3530   if (d)
3531     *(snat_det_session_t**)d = ses0;
3532   if (e)
3533     *(snat_det_map_t**)e = dm0;
3534   return next0;
3535 }
3536
3537 /**********************/
3538 /*** worker handoff ***/
3539 /**********************/
3540 static inline uword
3541 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3542                                       vlib_node_runtime_t * node,
3543                                       vlib_frame_t * frame,
3544                                       u8 is_output)
3545 {
3546   snat_main_t *sm = &snat_main;
3547   vlib_thread_main_t *tm = vlib_get_thread_main ();
3548   u32 n_left_from, *from, *to_next = 0;
3549   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3550   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3551     = 0;
3552   vlib_frame_queue_elt_t *hf = 0;
3553   vlib_frame_t *f = 0;
3554   int i;
3555   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3556   u32 next_worker_index = 0;
3557   u32 current_worker_index = ~0;
3558   u32 thread_index = vlib_get_thread_index ();
3559   u32 fq_index;
3560   u32 to_node_index;
3561
3562   ASSERT (vec_len (sm->workers));
3563
3564   if (is_output)
3565     {
3566       fq_index = sm->fq_in2out_output_index;
3567       to_node_index = sm->in2out_output_node_index;
3568     }
3569   else
3570     {
3571       fq_index = sm->fq_in2out_index;
3572       to_node_index = sm->in2out_node_index;
3573     }
3574
3575   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3576     {
3577       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3578
3579       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3580                                sm->first_worker_index + sm->num_workers - 1,
3581                                (vlib_frame_queue_t *) (~0));
3582     }
3583
3584   from = vlib_frame_vector_args (frame);
3585   n_left_from = frame->n_vectors;
3586
3587   while (n_left_from > 0)
3588     {
3589       u32 bi0;
3590       vlib_buffer_t *b0;
3591       u32 sw_if_index0;
3592       u32 rx_fib_index0;
3593       ip4_header_t * ip0;
3594       u8 do_handoff;
3595
3596       bi0 = from[0];
3597       from += 1;
3598       n_left_from -= 1;
3599
3600       b0 = vlib_get_buffer (vm, bi0);
3601
3602       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3603       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3604
3605       ip0 = vlib_buffer_get_current (b0);
3606
3607       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3608
3609       if (PREDICT_FALSE (next_worker_index != thread_index))
3610         {
3611           do_handoff = 1;
3612
3613           if (next_worker_index != current_worker_index)
3614             {
3615               if (hf)
3616                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3617
3618               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3619                                                       next_worker_index,
3620                                                       handoff_queue_elt_by_worker_index);
3621
3622               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3623               to_next_worker = &hf->buffer_index[hf->n_vectors];
3624               current_worker_index = next_worker_index;
3625             }
3626
3627           /* enqueue to correct worker thread */
3628           to_next_worker[0] = bi0;
3629           to_next_worker++;
3630           n_left_to_next_worker--;
3631
3632           if (n_left_to_next_worker == 0)
3633             {
3634               hf->n_vectors = VLIB_FRAME_SIZE;
3635               vlib_put_frame_queue_elt (hf);
3636               current_worker_index = ~0;
3637               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3638               hf = 0;
3639             }
3640         }
3641       else
3642         {
3643           do_handoff = 0;
3644           /* if this is 1st frame */
3645           if (!f)
3646             {
3647               f = vlib_get_frame_to_node (vm, to_node_index);
3648               to_next = vlib_frame_vector_args (f);
3649             }
3650
3651           to_next[0] = bi0;
3652           to_next += 1;
3653           f->n_vectors++;
3654         }
3655
3656       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3657                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3658         {
3659           snat_in2out_worker_handoff_trace_t *t =
3660             vlib_add_trace (vm, node, b0, sizeof (*t));
3661           t->next_worker_index = next_worker_index;
3662           t->do_handoff = do_handoff;
3663         }
3664     }
3665
3666   if (f)
3667     vlib_put_frame_to_node (vm, to_node_index, f);
3668
3669   if (hf)
3670     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3671
3672   /* Ship frames to the worker nodes */
3673   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3674     {
3675       if (handoff_queue_elt_by_worker_index[i])
3676         {
3677           hf = handoff_queue_elt_by_worker_index[i];
3678           /*
3679            * It works better to let the handoff node
3680            * rate-adapt, always ship the handoff queue element.
3681            */
3682           if (1 || hf->n_vectors == hf->last_n_vectors)
3683             {
3684               vlib_put_frame_queue_elt (hf);
3685               handoff_queue_elt_by_worker_index[i] = 0;
3686             }
3687           else
3688             hf->last_n_vectors = hf->n_vectors;
3689         }
3690       congested_handoff_queue_by_worker_index[i] =
3691         (vlib_frame_queue_t *) (~0);
3692     }
3693   hf = 0;
3694   current_worker_index = ~0;
3695   return frame->n_vectors;
3696 }
3697
3698 static uword
3699 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3700                                vlib_node_runtime_t * node,
3701                                vlib_frame_t * frame)
3702 {
3703   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3704 }
3705
3706 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3707   .function = snat_in2out_worker_handoff_fn,
3708   .name = "nat44-in2out-worker-handoff",
3709   .vector_size = sizeof (u32),
3710   .format_trace = format_snat_in2out_worker_handoff_trace,
3711   .type = VLIB_NODE_TYPE_INTERNAL,
3712
3713   .n_next_nodes = 1,
3714
3715   .next_nodes = {
3716     [0] = "error-drop",
3717   },
3718 };
3719
3720 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3721                               snat_in2out_worker_handoff_fn);
3722
3723 static uword
3724 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3725                                       vlib_node_runtime_t * node,
3726                                       vlib_frame_t * frame)
3727 {
3728   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3729 }
3730
3731 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3732   .function = snat_in2out_output_worker_handoff_fn,
3733   .name = "nat44-in2out-output-worker-handoff",
3734   .vector_size = sizeof (u32),
3735   .format_trace = format_snat_in2out_worker_handoff_trace,
3736   .type = VLIB_NODE_TYPE_INTERNAL,
3737
3738   .n_next_nodes = 1,
3739
3740   .next_nodes = {
3741     [0] = "error-drop",
3742   },
3743 };
3744
3745 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3746                               snat_in2out_output_worker_handoff_fn);
3747
3748 static_always_inline int
3749 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3750 {
3751   snat_address_t * ap;
3752   clib_bihash_kv_8_8_t kv, value;
3753   snat_session_key_t m_key;
3754
3755   vec_foreach (ap, sm->addresses)
3756     {
3757       if (ap->addr.as_u32 == dst_addr->as_u32)
3758         return 1;
3759     }
3760
3761   m_key.addr.as_u32 = dst_addr->as_u32;
3762   m_key.fib_index = sm->outside_fib_index;
3763   m_key.port = 0;
3764   m_key.protocol = 0;
3765   kv.key = m_key.as_u64;
3766   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3767     return 1;
3768
3769   return 0;
3770 }
3771
3772 static uword
3773 snat_hairpin_dst_fn (vlib_main_t * vm,
3774                      vlib_node_runtime_t * node,
3775                      vlib_frame_t * frame)
3776 {
3777   u32 n_left_from, * from, * to_next;
3778   snat_in2out_next_t next_index;
3779   u32 pkts_processed = 0;
3780   snat_main_t * sm = &snat_main;
3781
3782   from = vlib_frame_vector_args (frame);
3783   n_left_from = frame->n_vectors;
3784   next_index = node->cached_next_index;
3785
3786   while (n_left_from > 0)
3787     {
3788       u32 n_left_to_next;
3789
3790       vlib_get_next_frame (vm, node, next_index,
3791                            to_next, n_left_to_next);
3792
3793       while (n_left_from > 0 && n_left_to_next > 0)
3794         {
3795           u32 bi0;
3796           vlib_buffer_t * b0;
3797           u32 next0;
3798           ip4_header_t * ip0;
3799           u32 proto0;
3800
3801           /* speculatively enqueue b0 to the current next frame */
3802           bi0 = from[0];
3803           to_next[0] = bi0;
3804           from += 1;
3805           to_next += 1;
3806           n_left_from -= 1;
3807           n_left_to_next -= 1;
3808
3809           b0 = vlib_get_buffer (vm, bi0);
3810           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3811           ip0 = vlib_buffer_get_current (b0);
3812
3813           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3814
3815           vnet_buffer (b0)->snat.flags = 0;
3816           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3817             {
3818               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3819                 {
3820                   udp_header_t * udp0 = ip4_next_header (ip0);
3821                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3822
3823                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3824                 }
3825               else if (proto0 == SNAT_PROTOCOL_ICMP)
3826                 {
3827                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3828
3829                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3830                 }
3831               else
3832                 {
3833                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3834                 }
3835
3836               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3837             }
3838
3839           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3840
3841           /* verify speculative enqueue, maybe switch current next frame */
3842           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3843                                            to_next, n_left_to_next,
3844                                            bi0, next0);
3845          }
3846
3847       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3848     }
3849
3850   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3851                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3852                                pkts_processed);
3853   return frame->n_vectors;
3854 }
3855
3856 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3857   .function = snat_hairpin_dst_fn,
3858   .name = "nat44-hairpin-dst",
3859   .vector_size = sizeof (u32),
3860   .type = VLIB_NODE_TYPE_INTERNAL,
3861   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3862   .error_strings = snat_in2out_error_strings,
3863   .n_next_nodes = 2,
3864   .next_nodes = {
3865     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3866     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3867   },
3868 };
3869
3870 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3871                               snat_hairpin_dst_fn);
3872
3873 static uword
3874 snat_hairpin_src_fn (vlib_main_t * vm,
3875                      vlib_node_runtime_t * node,
3876                      vlib_frame_t * frame)
3877 {
3878   u32 n_left_from, * from, * to_next;
3879   snat_in2out_next_t next_index;
3880   u32 pkts_processed = 0;
3881   snat_main_t *sm = &snat_main;
3882
3883   from = vlib_frame_vector_args (frame);
3884   n_left_from = frame->n_vectors;
3885   next_index = node->cached_next_index;
3886
3887   while (n_left_from > 0)
3888     {
3889       u32 n_left_to_next;
3890
3891       vlib_get_next_frame (vm, node, next_index,
3892                            to_next, n_left_to_next);
3893
3894       while (n_left_from > 0 && n_left_to_next > 0)
3895         {
3896           u32 bi0;
3897           vlib_buffer_t * b0;
3898           u32 next0;
3899           snat_interface_t *i;
3900           u32 sw_if_index0;
3901
3902           /* speculatively enqueue b0 to the current next frame */
3903           bi0 = from[0];
3904           to_next[0] = bi0;
3905           from += 1;
3906           to_next += 1;
3907           n_left_from -= 1;
3908           n_left_to_next -= 1;
3909
3910           b0 = vlib_get_buffer (vm, bi0);
3911           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3912           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3913
3914           pool_foreach (i, sm->output_feature_interfaces,
3915           ({
3916             /* Only packets from NAT inside interface */
3917             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3918               {
3919                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3920                                     SNAT_FLAG_HAIRPINNING))
3921                   {
3922                     if (PREDICT_TRUE (sm->num_workers > 1))
3923                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3924                     else
3925                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3926                   }
3927                 break;
3928               }
3929           }));
3930
3931           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3932
3933           /* verify speculative enqueue, maybe switch current next frame */
3934           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3935                                            to_next, n_left_to_next,
3936                                            bi0, next0);
3937          }
3938
3939       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3940     }
3941
3942   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3943                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3944                                pkts_processed);
3945   return frame->n_vectors;
3946 }
3947
3948 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3949   .function = snat_hairpin_src_fn,
3950   .name = "nat44-hairpin-src",
3951   .vector_size = sizeof (u32),
3952   .type = VLIB_NODE_TYPE_INTERNAL,
3953   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3954   .error_strings = snat_in2out_error_strings,
3955   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3956   .next_nodes = {
3957      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3958      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3959      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3960      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3961   },
3962 };
3963
3964 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3965                               snat_hairpin_src_fn);
3966
3967 static uword
3968 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3969                                 vlib_node_runtime_t * node,
3970                                 vlib_frame_t * frame)
3971 {
3972   u32 n_left_from, * from, * to_next;
3973   snat_in2out_next_t next_index;
3974   u32 pkts_processed = 0;
3975   snat_main_t * sm = &snat_main;
3976   u32 stats_node_index;
3977
3978   stats_node_index = snat_in2out_fast_node.index;
3979
3980   from = vlib_frame_vector_args (frame);
3981   n_left_from = frame->n_vectors;
3982   next_index = node->cached_next_index;
3983
3984   while (n_left_from > 0)
3985     {
3986       u32 n_left_to_next;
3987
3988       vlib_get_next_frame (vm, node, next_index,
3989                            to_next, n_left_to_next);
3990
3991       while (n_left_from > 0 && n_left_to_next > 0)
3992         {
3993           u32 bi0;
3994           vlib_buffer_t * b0;
3995           u32 next0;
3996           u32 sw_if_index0;
3997           ip4_header_t * ip0;
3998           ip_csum_t sum0;
3999           u32 new_addr0, old_addr0;
4000           u16 old_port0, new_port0;
4001           udp_header_t * udp0;
4002           tcp_header_t * tcp0;
4003           icmp46_header_t * icmp0;
4004           snat_session_key_t key0, sm0;
4005           u32 proto0;
4006           u32 rx_fib_index0;
4007
4008           /* speculatively enqueue b0 to the current next frame */
4009           bi0 = from[0];
4010           to_next[0] = bi0;
4011           from += 1;
4012           to_next += 1;
4013           n_left_from -= 1;
4014           n_left_to_next -= 1;
4015
4016           b0 = vlib_get_buffer (vm, bi0);
4017           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4018
4019           ip0 = vlib_buffer_get_current (b0);
4020           udp0 = ip4_next_header (ip0);
4021           tcp0 = (tcp_header_t *) udp0;
4022           icmp0 = (icmp46_header_t *) udp0;
4023
4024           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4025           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4026
4027           if (PREDICT_FALSE(ip0->ttl == 1))
4028             {
4029               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4030               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4031                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4032                                            0);
4033               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4034               goto trace0;
4035             }
4036
4037           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4038
4039           if (PREDICT_FALSE (proto0 == ~0))
4040               goto trace0;
4041
4042           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4043             {
4044               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4045                                   rx_fib_index0, node, next0, ~0, 0, 0);
4046               goto trace0;
4047             }
4048
4049           key0.addr = ip0->src_address;
4050           key0.protocol = proto0;
4051           key0.port = udp0->src_port;
4052           key0.fib_index = rx_fib_index0;
4053
4054           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
4055             {
4056               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4057               next0= SNAT_IN2OUT_NEXT_DROP;
4058               goto trace0;
4059             }
4060
4061           new_addr0 = sm0.addr.as_u32;
4062           new_port0 = sm0.port;
4063           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4064           old_addr0 = ip0->src_address.as_u32;
4065           ip0->src_address.as_u32 = new_addr0;
4066
4067           sum0 = ip0->checksum;
4068           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4069                                  ip4_header_t,
4070                                  src_address /* changed member */);
4071           ip0->checksum = ip_csum_fold (sum0);
4072
4073           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4074             {
4075               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4076                 {
4077                   old_port0 = tcp0->src_port;
4078                   tcp0->src_port = new_port0;
4079
4080                   sum0 = tcp0->checksum;
4081                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4082                                          ip4_header_t,
4083                                          dst_address /* changed member */);
4084                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4085                                          ip4_header_t /* cheat */,
4086                                          length /* changed member */);
4087                   tcp0->checksum = ip_csum_fold(sum0);
4088                 }
4089               else
4090                 {
4091                   old_port0 = udp0->src_port;
4092                   udp0->src_port = new_port0;
4093                   udp0->checksum = 0;
4094                 }
4095             }
4096           else
4097             {
4098               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4099                 {
4100                   sum0 = tcp0->checksum;
4101                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4102                                          ip4_header_t,
4103                                          dst_address /* changed member */);
4104                   tcp0->checksum = ip_csum_fold(sum0);
4105                 }
4106             }
4107
4108           /* Hairpinning */
4109           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4110
4111         trace0:
4112           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4113                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4114             {
4115               snat_in2out_trace_t *t =
4116                  vlib_add_trace (vm, node, b0, sizeof (*t));
4117               t->sw_if_index = sw_if_index0;
4118               t->next_index = next0;
4119             }
4120
4121           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4122
4123           /* verify speculative enqueue, maybe switch current next frame */
4124           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4125                                            to_next, n_left_to_next,
4126                                            bi0, next0);
4127         }
4128
4129       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4130     }
4131
4132   vlib_node_increment_counter (vm, stats_node_index,
4133                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4134                                pkts_processed);
4135   return frame->n_vectors;
4136 }
4137
4138
4139 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
4140   .function = snat_in2out_fast_static_map_fn,
4141   .name = "nat44-in2out-fast",
4142   .vector_size = sizeof (u32),
4143   .format_trace = format_snat_in2out_fast_trace,
4144   .type = VLIB_NODE_TYPE_INTERNAL,
4145
4146   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4147   .error_strings = snat_in2out_error_strings,
4148
4149   .runtime_data_bytes = sizeof (snat_runtime_t),
4150
4151   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4152
4153   /* edit / add dispositions here */
4154   .next_nodes = {
4155     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4156     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4157     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4158     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4159     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4160   },
4161 };
4162
4163 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);