src/plugins/nat/in2out.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vlib/vlib.h>
  17 #include <vnet/vnet.h>
  18 #include <vnet/pg/pg.h>
  19 #include <vnet/handoff.h>
  20
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/ethernet/ethernet.h>
  23 #include <vnet/fib/ip4_fib.h>
  24 #include <nat/nat.h>
  25 #include <nat/nat_ipfix_logging.h>
  26 #include <nat/nat_det.h>
  27 #include <nat/nat_reass.h>
  28
  29 #include <vppinfra/hash.h>
  30 #include <vppinfra/error.h>
  31 #include <vppinfra/elog.h>
  32
  33 typedef struct {
  34   u32 sw_if_index;
  35   u32 next_index;
  36   u32 session_index;
  37   u32 is_slow_path;
  38 } snat_in2out_trace_t;
  39
  40 typedef struct {
  41   u32 next_worker_index;
  42   u8 do_handoff;
  43 } snat_in2out_worker_handoff_trace_t;
  44
  45 /* packet trace format function */
  46 static u8 * format_snat_in2out_trace (u8 * s, va_list * args)
  47 {
  48   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  49   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  50   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  51   char * tag;
  52
  53   tag = t->is_slow_path ? "NAT44_IN2OUT_SLOW_PATH" : "NAT44_IN2OUT_FAST_PATH";
  54
  55   s = format (s, "%s: sw_if_index %d, next index %d, session %d", tag,
  56               t->sw_if_index, t->next_index, t->session_index);
  57
  58   return s;
  59 }
  60
  61 static u8 * format_snat_in2out_fast_trace (u8 * s, va_list * args)
  62 {
  63   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  64   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  65   snat_in2out_trace_t * t = va_arg (*args, snat_in2out_trace_t *);
  66
  67   s = format (s, "NAT44_IN2OUT_FAST: sw_if_index %d, next index %d",
  68               t->sw_if_index, t->next_index);
  69
  70   return s;
  71 }
  72
  73 static u8 * format_snat_in2out_worker_handoff_trace (u8 * s, va_list * args)
  74 {
  75   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  76   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  77   snat_in2out_worker_handoff_trace_t * t =
  78     va_arg (*args, snat_in2out_worker_handoff_trace_t *);
  79   char * m;
  80
  81   m = t->do_handoff ? "next worker" : "same worker";
  82   s = format (s, "NAT44_IN2OUT_WORKER_HANDOFF: %s %d", m, t->next_worker_index);
  83
  84   return s;
  85 }
  86
  87 typedef struct {
  88   u32 sw_if_index;
  89   u32 next_index;
  90   u8 cached;
  91 } nat44_in2out_reass_trace_t;
  92
  93 static u8 * format_nat44_in2out_reass_trace (u8 * s, va_list * args)
  94 {
  95   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  96   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  97   nat44_in2out_reass_trace_t * t = va_arg (*args, nat44_in2out_reass_trace_t *);
  98
  99   s = format (s, "NAT44_IN2OUT_REASS: sw_if_index %d, next index %d, status %s",
 100               t->sw_if_index, t->next_index,
 101               t->cached ? "cached" : "translated");
 102
 103   return s;
 104 }
 105
 106 vlib_node_registration_t snat_in2out_node;
 107 vlib_node_registration_t snat_in2out_slowpath_node;
 108 vlib_node_registration_t snat_in2out_fast_node;
 109 vlib_node_registration_t snat_in2out_worker_handoff_node;
 110 vlib_node_registration_t snat_det_in2out_node;
 111 vlib_node_registration_t snat_in2out_output_node;
 112 vlib_node_registration_t snat_in2out_output_slowpath_node;
 113 vlib_node_registration_t snat_in2out_output_worker_handoff_node;
 114 vlib_node_registration_t snat_hairpin_dst_node;
 115 vlib_node_registration_t snat_hairpin_src_node;
 116 vlib_node_registration_t nat44_hairpinning_node;
 117 vlib_node_registration_t nat44_in2out_reass_node;
 118
 119
 120 #define foreach_snat_in2out_error                       \
 121 _(UNSUPPORTED_PROTOCOL, "Unsupported protocol")         \
 122 _(IN2OUT_PACKETS, "Good in2out packets processed")      \
 123 _(OUT_OF_PORTS, "Out of ports")                         \
 124 _(BAD_OUTSIDE_FIB, "Outside VRF ID not found")          \
 125 _(BAD_ICMP_TYPE, "unsupported ICMP type")               \
 126 _(NO_TRANSLATION, "No translation")                     \
 127 _(MAX_SESSIONS_EXCEEDED, "Maximum sessions exceeded")   \
 128 _(DROP_FRAGMENT, "Drop fragment")                       \
 129 _(MAX_REASS, "Maximum reassemblies exceeded")           \
 130 _(MAX_FRAG, "Maximum fragments per reassembly exceeded")
 131
 132 typedef enum {
 133 #define _(sym,str) SNAT_IN2OUT_ERROR_##sym,
 134   foreach_snat_in2out_error
 135 #undef _
 136   SNAT_IN2OUT_N_ERROR,
 137 } snat_in2out_error_t;
 138
 139 static char * snat_in2out_error_strings[] = {
 140 #define _(sym,string) string,
 141   foreach_snat_in2out_error
 142 #undef _
 143 };
 144
 145 typedef enum {
 146   SNAT_IN2OUT_NEXT_LOOKUP,
 147   SNAT_IN2OUT_NEXT_DROP,
 148   SNAT_IN2OUT_NEXT_ICMP_ERROR,
 149   SNAT_IN2OUT_NEXT_SLOW_PATH,
 150   SNAT_IN2OUT_NEXT_REASS,
 151   SNAT_IN2OUT_N_NEXT,
 152 } snat_in2out_next_t;
 153
 154 typedef enum {
 155   SNAT_HAIRPIN_SRC_NEXT_DROP,
 156   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT,
 157   SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH,
 158   SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT,
 159   SNAT_HAIRPIN_SRC_N_NEXT,
 160 } snat_hairpin_next_t;
 161
 162 /**
 163  * @brief Check if packet should be translated
 164  *
 165  * Packets aimed at outside interface and external address with active session
 166  * should be translated.
 167  *
 168  * @param sm            NAT main
 169  * @param rt            NAT runtime data
 170  * @param sw_if_index0  index of the inside interface
 171  * @param ip0           IPv4 header
 172  * @param proto0        NAT protocol
 173  * @param rx_fib_index0 RX FIB index
 174  *
 175  * @returns 0 if packet should be translated otherwise 1
 176  */
 177 static inline int
 178 snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
 179                          u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 180                          u32 rx_fib_index0)
 181 {
 182   if (sm->out2in_dpo)
 183     return 0;
 184
 185   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
 186   fib_prefix_t pfx = {
 187     .fp_proto = FIB_PROTOCOL_IP4,
 188     .fp_len = 32,
 189     .fp_addr = {
 190         .ip4.as_u32 = ip0->dst_address.as_u32,
 191     },
 192   };
 193
 194   /* Don't NAT packet aimed at the intfc address */
 195   if (PREDICT_FALSE(is_interface_addr(sm, node, sw_if_index0,
 196                                       ip0->dst_address.as_u32)))
 197     return 1;
 198
 199   fei = fib_table_lookup (rx_fib_index0, &pfx);
 200   if (FIB_NODE_INDEX_INVALID != fei)
 201     {
 202       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
 203       if (sw_if_index == ~0)
 204         {
 205           fei = fib_table_lookup (sm->outside_fib_index, &pfx);
 206           if (FIB_NODE_INDEX_INVALID != fei)
 207             sw_if_index = fib_entry_get_resolving_interface (fei);
 208         }
 209       snat_interface_t *i;
 210       pool_foreach (i, sm->interfaces,
 211       ({
 212         /* NAT packet aimed at outside interface */
 213         if ((nat_interface_is_outside(i)) && (sw_if_index == i->sw_if_index))
 214           return 0;
 215       }));
 216     }
 217
 218   return 1;
 219 }
 220
 221 static inline int
 222 snat_not_translate (snat_main_t * sm, vlib_node_runtime_t *node,
 223                     u32 sw_if_index0, ip4_header_t * ip0, u32 proto0,
 224                     u32 rx_fib_index0, u32 thread_index)
 225 {
 226   udp_header_t * udp0 = ip4_next_header (ip0);
 227   snat_session_key_t key0, sm0;
 228   clib_bihash_kv_8_8_t kv0, value0;
 229
 230   key0.addr = ip0->dst_address;
 231   key0.port = udp0->dst_port;
 232   key0.protocol = proto0;
 233   key0.fib_index = sm->outside_fib_index;
 234   kv0.key = key0.as_u64;
 235
 236   /* NAT packet aimed at external address if */
 237   /* has active sessions */
 238   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 239                               &value0))
 240     {
 241       /* or is static mappings */
 242       if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 243         return 0;
 244     }
 245   else
 246     return 0;
 247
 248   if (sm->forwarding_enabled)
 249     return 1;
 250
 251   return snat_not_translate_fast(sm, node, sw_if_index0, ip0, proto0,
 252                                  rx_fib_index0);
 253 }
 254
 255 static inline int
 256 nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
 257                                   u32 proto0, u16 src_port, u16 dst_port,
 258                                   u32 thread_index, u32 sw_if_index)
 259 {
 260   snat_session_key_t key0;
 261   clib_bihash_kv_8_8_t kv0, value0;
 262   snat_interface_t *i;
 263
 264   /* src NAT check */
 265   key0.addr = ip0->src_address;
 266   key0.port = src_port;
 267   key0.protocol = proto0;
 268   key0.fib_index = sm->outside_fib_index;
 269   kv0.key = key0.as_u64;
 270
 271   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 272                                &value0))
 273     return 1;
 274
 275   /* dst NAT check */
 276   key0.addr = ip0->dst_address;
 277   key0.port = dst_port;
 278   key0.protocol = proto0;
 279   key0.fib_index = sm->inside_fib_index;
 280   kv0.key = key0.as_u64;
 281   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 282                                &value0))
 283   {
 284     /* hairpinning */
 285     pool_foreach (i, sm->output_feature_interfaces,
 286     ({
 287       if ((nat_interface_is_inside(i)) && (sw_if_index == i->sw_if_index))
 288         return 0;
 289     }));
 290     return 1;
 291   }
 292
 293   return 0;
 294 }
 295
 296 static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
 297                       ip4_header_t * ip0,
 298                       u32 rx_fib_index0,
 299                       snat_session_key_t * key0,
 300                       snat_session_t ** sessionp,
 301                       vlib_node_runtime_t * node,
 302                       u32 next0,
 303                       u32 thread_index)
 304 {
 305   snat_user_t *u;
 306   snat_session_t *s;
 307   clib_bihash_kv_8_8_t kv0;
 308   snat_session_key_t key1;
 309   u32 address_index = ~0;
 310   u32 outside_fib_index;
 311   uword * p;
 312   udp_header_t * udp0 = ip4_next_header (ip0);
 313   u8 is_sm = 0;
 314
 315   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
 316     {
 317       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
 318       nat_ipfix_logging_max_sessions(sm->max_translations);
 319       return SNAT_IN2OUT_NEXT_DROP;
 320     }
 321
 322   p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
 323   if (! p)
 324     {
 325       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
 326       return SNAT_IN2OUT_NEXT_DROP;
 327     }
 328   outside_fib_index = p[0];
 329
 330   key1.protocol = key0->protocol;
 331
 332   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
 333                               thread_index);
 334   if (!u)
 335     {
 336       clib_warning ("create NAT user failed");
 337       return SNAT_IN2OUT_NEXT_DROP;
 338     }
 339
 340   /* First try to match static mapping by local address and port */
 341   if (snat_static_mapping_match (sm, *key0, &key1, 0, 0, 0, 0))
 342     {
 343       /* Try to create dynamic translation */
 344       if (snat_alloc_outside_address_and_port (sm->addresses, rx_fib_index0,
 345                                                thread_index, &key1,
 346                                                &address_index,
 347                                                sm->port_per_thread,
 348                                                sm->per_thread_data[thread_index].snat_thread_index))
 349         {
 350           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
 351           return SNAT_IN2OUT_NEXT_DROP;
 352         }
 353     }
 354   else
 355     is_sm = 1;
 356
 357   s = nat_session_alloc_or_recycle (sm, u, thread_index);
 358   if (!s)
 359     {
 360       clib_warning ("create NAT session failed");
 361       return SNAT_IN2OUT_NEXT_DROP;
 362     }
 363
 364   if (is_sm)
 365     s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
 366   user_session_increment (sm, u, is_sm);
 367   s->outside_address_index = address_index;
 368   s->in2out = *key0;
 369   s->out2in = key1;
 370   s->out2in.protocol = key0->protocol;
 371   s->out2in.fib_index = outside_fib_index;
 372   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
 373   s->ext_host_port = udp0->dst_port;
 374   *sessionp = s;
 375
 376   /* Add to translation hashes */
 377   kv0.key = s->in2out.as_u64;
 378   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 379   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 380                                1 /* is_add */))
 381       clib_warning ("in2out key add failed");
 382
 383   kv0.key = s->out2in.as_u64;
 384   kv0.value = s - sm->per_thread_data[thread_index].sessions;
 385
 386   if (clib_bihash_add_del_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
 387                                1 /* is_add */))
 388       clib_warning ("out2in key add failed");
 389
 390   /* log NAT event */
 391   snat_ipfix_logging_nat44_ses_create(s->in2out.addr.as_u32,
 392                                       s->out2in.addr.as_u32,
 393                                       s->in2out.protocol,
 394                                       s->in2out.port,
 395                                       s->out2in.port,
 396                                       s->in2out.fib_index);
 397   return next0;
 398 }
 399
 400 static_always_inline
 401 snat_in2out_error_t icmp_get_key(ip4_header_t *ip0,
 402                                  snat_session_key_t *p_key0)
 403 {
 404   icmp46_header_t *icmp0;
 405   snat_session_key_t key0;
 406   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 407   ip4_header_t *inner_ip0 = 0;
 408   void *l4_header = 0;
 409   icmp46_header_t *inner_icmp0;
 410
 411   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 412   echo0 = (icmp_echo_header_t *)(icmp0+1);
 413
 414   if (!icmp_is_error_message (icmp0))
 415     {
 416       key0.protocol = SNAT_PROTOCOL_ICMP;
 417       key0.addr = ip0->src_address;
 418       key0.port = echo0->identifier;
 419     }
 420   else
 421     {
 422       inner_ip0 = (ip4_header_t *)(echo0+1);
 423       l4_header = ip4_next_header (inner_ip0);
 424       key0.protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
 425       key0.addr = inner_ip0->dst_address;
 426       switch (key0.protocol)
 427         {
 428         case SNAT_PROTOCOL_ICMP:
 429           inner_icmp0 = (icmp46_header_t*)l4_header;
 430           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 431           key0.port = inner_echo0->identifier;
 432           break;
 433         case SNAT_PROTOCOL_UDP:
 434         case SNAT_PROTOCOL_TCP:
 435           key0.port = ((tcp_udp_header_t*)l4_header)->dst_port;
 436           break;
 437         default:
 438           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 439         }
 440     }
 441   *p_key0 = key0;
 442   return -1; /* success */
 443 }
 444
 445 static_always_inline int
 446 icmp_get_ed_key(ip4_header_t *ip0, nat_ed_ses_key_t *p_key0)
 447 {
 448   icmp46_header_t *icmp0;
 449   nat_ed_ses_key_t key0;
 450   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 451   ip4_header_t *inner_ip0 = 0;
 452   void *l4_header = 0;
 453   icmp46_header_t *inner_icmp0;
 454
 455   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 456   echo0 = (icmp_echo_header_t *)(icmp0+1);
 457
 458   if (!icmp_is_error_message (icmp0))
 459     {
 460       key0.proto = IP_PROTOCOL_ICMP;
 461       key0.l_addr = ip0->src_address;
 462       key0.r_addr = ip0->dst_address;
 463       key0.l_port = key0.r_port = echo0->identifier;
 464     }
 465   else
 466     {
 467       inner_ip0 = (ip4_header_t *)(echo0+1);
 468       l4_header = ip4_next_header (inner_ip0);
 469       key0.proto = inner_ip0->protocol;
 470       key0.r_addr = inner_ip0->src_address;
 471       key0.l_addr = inner_ip0->dst_address;
 472       switch (ip_proto_to_snat_proto (inner_ip0->protocol))
 473         {
 474         case SNAT_PROTOCOL_ICMP:
 475           inner_icmp0 = (icmp46_header_t*)l4_header;
 476           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 477           key0.r_port = key0.l_port = inner_echo0->identifier;
 478           break;
 479         case SNAT_PROTOCOL_UDP:
 480         case SNAT_PROTOCOL_TCP:
 481           key0.l_port = ((tcp_udp_header_t*)l4_header)->dst_port;
 482           key0.r_port = ((tcp_udp_header_t*)l4_header)->src_port;
 483           break;
 484         default:
 485           return SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL;
 486         }
 487     }
 488   *p_key0 = key0;
 489   return 0;
 490 }
 491
 492 static inline int
 493 nat_not_translate_output_feature_fwd (snat_main_t * sm, ip4_header_t * ip,
 494                                       u32 thread_index)
 495 {
 496   nat_ed_ses_key_t key;
 497   clib_bihash_kv_16_8_t kv, value;
 498   udp_header_t *udp;
 499   snat_session_t *s = 0;
 500   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
 501
 502   if (!sm->forwarding_enabled)
 503     return 0;
 504
 505   if (ip->protocol == IP_PROTOCOL_ICMP)
 506     {
 507       if (icmp_get_ed_key (ip, &key))
 508         return 0;
 509     }
 510   else if (ip->protocol == IP_PROTOCOL_UDP || ip->protocol == IP_PROTOCOL_TCP)
 511     {
 512       udp = ip4_next_header(ip);
 513       key.l_addr = ip->src_address;
 514       key.r_addr = ip->dst_address;
 515       key.proto = ip->protocol;
 516       key.r_port = udp->dst_port;
 517       key.l_port = udp->src_port;
 518     }
 519   else
 520     {
 521       key.l_addr = ip->src_address;
 522       key.r_addr = ip->dst_address;
 523       key.proto = ip->protocol;
 524       key.l_port = key.r_port = 0;
 525     }
 526   key.fib_index = 0;
 527   kv.key[0] = key.as_u64[0];
 528   kv.key[1] = key.as_u64[1];
 529
 530   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &kv, &value))
 531     {
 532       s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value.value);
 533       if (is_fwd_bypass_session (s))
 534         {
 535           /* Per-user LRU list maintenance */
 536           clib_dlist_remove (tsm->list_pool, s->per_user_index);
 537           clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
 538                               s->per_user_index);
 539           return 1;
 540         }
 541       else
 542         return 0;
 543     }
 544
 545   return 0;
 546 }
 547
 548 /**
 549  * Get address and port values to be used for ICMP packet translation
 550  * and create session if needed
 551  *
 552  * @param[in,out] sm             NAT main
 553  * @param[in,out] node           NAT node runtime
 554  * @param[in] thread_index       thread index
 555  * @param[in,out] b0             buffer containing packet to be translated
 556  * @param[out] p_proto           protocol used for matching
 557  * @param[out] p_value           address and port after NAT translation
 558  * @param[out] p_dont_translate  if packet should not be translated
 559  * @param d                      optional parameter
 560  * @param e                      optional parameter
 561  */
 562 u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node,
 563                            u32 thread_index, vlib_buffer_t *b0,
 564                            ip4_header_t *ip0, u8 *p_proto,
 565                            snat_session_key_t *p_value,
 566                            u8 *p_dont_translate, void *d, void *e)
 567 {
 568   icmp46_header_t *icmp0;
 569   u32 sw_if_index0;
 570   u32 rx_fib_index0;
 571   snat_session_key_t key0;
 572   snat_session_t *s0 = 0;
 573   u8 dont_translate = 0;
 574   clib_bihash_kv_8_8_t kv0, value0;
 575   u32 next0 = ~0;
 576   int err;
 577
 578   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 579   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 580   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 581
 582   err = icmp_get_key (ip0, &key0);
 583   if (err != -1)
 584     {
 585       b0->error = node->errors[err];
 586       next0 = SNAT_IN2OUT_NEXT_DROP;
 587       goto out;
 588     }
 589   key0.fib_index = rx_fib_index0;
 590
 591   kv0.key = key0.as_u64;
 592
 593   if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
 594                               &value0))
 595     {
 596       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] != ~0)
 597         {
 598           if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
 599               ip0, SNAT_PROTOCOL_ICMP, key0.port, key0.port, thread_index, sw_if_index0)))
 600             {
 601               dont_translate = 1;
 602               goto out;
 603             }
 604         }
 605       else
 606         {
 607           if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
 608               ip0, SNAT_PROTOCOL_ICMP, rx_fib_index0, thread_index)))
 609             {
 610               dont_translate = 1;
 611               goto out;
 612             }
 613         }
 614
 615       if (PREDICT_FALSE(icmp_is_error_message (icmp0)))
 616         {
 617           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 618           next0 = SNAT_IN2OUT_NEXT_DROP;
 619           goto out;
 620         }
 621
 622       next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
 623                          &s0, node, next0, thread_index);
 624
 625       if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
 626         goto out;
 627     }
 628   else
 629     {
 630       if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 631                         icmp0->type != ICMP4_echo_reply &&
 632                         !icmp_is_error_message (icmp0)))
 633         {
 634           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 635           next0 = SNAT_IN2OUT_NEXT_DROP;
 636           goto out;
 637         }
 638
 639       if (PREDICT_FALSE (value0.value == ~0ULL))
 640         {
 641           nat_ed_ses_key_t key;
 642           clib_bihash_kv_16_8_t s_kv, s_value;
 643
 644           key.as_u64[0] = 0;
 645           key.as_u64[1] = 0;
 646           if (icmp_get_ed_key (ip0, &key))
 647             {
 648               b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
 649               next0 = SNAT_IN2OUT_NEXT_DROP;
 650               goto out;
 651             }
 652           key.fib_index = rx_fib_index0;
 653           s_kv.key[0] = key.as_u64[0];
 654           s_kv.key[1] = key.as_u64[1];
 655           if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
 656             s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 657                                     s_value.value);
 658           else
 659            {
 660               next0 = SNAT_IN2OUT_NEXT_DROP;
 661               goto out;
 662            }
 663         }
 664       else
 665         s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions,
 666                                 value0.value);
 667     }
 668
 669 out:
 670   *p_proto = key0.protocol;
 671   if (s0)
 672     *p_value = s0->out2in;
 673   *p_dont_translate = dont_translate;
 674   if (d)
 675     *(snat_session_t**)d = s0;
 676   return next0;
 677 }
 678
 679 /**
 680  * Get address and port values to be used for ICMP packet translation
 681  *
 682  * @param[in] sm                 NAT main
 683  * @param[in,out] node           NAT node runtime
 684  * @param[in] thread_index       thread index
 685  * @param[in,out] b0             buffer containing packet to be translated
 686  * @param[out] p_proto           protocol used for matching
 687  * @param[out] p_value           address and port after NAT translation
 688  * @param[out] p_dont_translate  if packet should not be translated
 689  * @param d                      optional parameter
 690  * @param e                      optional parameter
 691  */
 692 u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node,
 693                            u32 thread_index, vlib_buffer_t *b0,
 694                            ip4_header_t *ip0, u8 *p_proto,
 695                            snat_session_key_t *p_value,
 696                            u8 *p_dont_translate, void *d, void *e)
 697 {
 698   icmp46_header_t *icmp0;
 699   u32 sw_if_index0;
 700   u32 rx_fib_index0;
 701   snat_session_key_t key0;
 702   snat_session_key_t sm0;
 703   u8 dont_translate = 0;
 704   u8 is_addr_only;
 705   u32 next0 = ~0;
 706   int err;
 707
 708   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
 709   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 710   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 711
 712   err = icmp_get_key (ip0, &key0);
 713   if (err != -1)
 714     {
 715       b0->error = node->errors[err];
 716       next0 = SNAT_IN2OUT_NEXT_DROP;
 717       goto out2;
 718     }
 719   key0.fib_index = rx_fib_index0;
 720
 721   if (snat_static_mapping_match(sm, key0, &sm0, 0, &is_addr_only, 0, 0))
 722     {
 723       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
 724           IP_PROTOCOL_ICMP, rx_fib_index0)))
 725         {
 726           dont_translate = 1;
 727           goto out;
 728         }
 729
 730       if (icmp_is_error_message (icmp0))
 731         {
 732           next0 = SNAT_IN2OUT_NEXT_DROP;
 733           goto out;
 734         }
 735
 736       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
 737       next0 = SNAT_IN2OUT_NEXT_DROP;
 738       goto out;
 739     }
 740
 741   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
 742                     (icmp0->type != ICMP4_echo_reply || !is_addr_only) &&
 743                     !icmp_is_error_message (icmp0)))
 744     {
 745       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
 746       next0 = SNAT_IN2OUT_NEXT_DROP;
 747       goto out;
 748     }
 749
 750 out:
 751   *p_value = sm0;
 752 out2:
 753   *p_proto = key0.protocol;
 754   *p_dont_translate = dont_translate;
 755   return next0;
 756 }
 757
 758 static inline u32 icmp_in2out (snat_main_t *sm,
 759                                vlib_buffer_t * b0,
 760                                ip4_header_t * ip0,
 761                                icmp46_header_t * icmp0,
 762                                u32 sw_if_index0,
 763                                u32 rx_fib_index0,
 764                                vlib_node_runtime_t * node,
 765                                u32 next0,
 766                                u32 thread_index,
 767                                void *d,
 768                                void *e)
 769 {
 770   snat_session_key_t sm0;
 771   u8 protocol;
 772   icmp_echo_header_t *echo0, *inner_echo0 = 0;
 773   ip4_header_t *inner_ip0;
 774   void *l4_header = 0;
 775   icmp46_header_t *inner_icmp0;
 776   u8 dont_translate;
 777   u32 new_addr0, old_addr0;
 778   u16 old_id0, new_id0;
 779   ip_csum_t sum0;
 780   u16 checksum0;
 781   u32 next0_tmp;
 782
 783   echo0 = (icmp_echo_header_t *)(icmp0+1);
 784
 785   next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, ip0,
 786                                        &protocol, &sm0, &dont_translate, d, e);
 787   if (next0_tmp != ~0)
 788     next0 = next0_tmp;
 789   if (next0 == SNAT_IN2OUT_NEXT_DROP || dont_translate)
 790     goto out;
 791
 792   sum0 = ip_incremental_checksum (0, icmp0,
 793                                   ntohs(ip0->length) - ip4_header_bytes (ip0));
 794   checksum0 = ~ip_csum_fold (sum0);
 795   if (PREDICT_FALSE(checksum0 != 0 && checksum0 != 0xffff))
 796     {
 797       next0 = SNAT_IN2OUT_NEXT_DROP;
 798       goto out;
 799     }
 800
 801   old_addr0 = ip0->src_address.as_u32;
 802   new_addr0 = ip0->src_address.as_u32 = sm0.addr.as_u32;
 803   if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == ~0)
 804     vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 805
 806   sum0 = ip0->checksum;
 807   sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 808                          src_address /* changed member */);
 809   ip0->checksum = ip_csum_fold (sum0);
 810
 811   if (icmp0->checksum == 0)
 812     icmp0->checksum = 0xffff;
 813
 814   if (!icmp_is_error_message (icmp0))
 815     {
 816       new_id0 = sm0.port;
 817       if (PREDICT_FALSE(new_id0 != echo0->identifier))
 818         {
 819           old_id0 = echo0->identifier;
 820           new_id0 = sm0.port;
 821           echo0->identifier = new_id0;
 822
 823           sum0 = icmp0->checksum;
 824           sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 825                                  identifier);
 826           icmp0->checksum = ip_csum_fold (sum0);
 827         }
 828     }
 829   else
 830     {
 831       inner_ip0 = (ip4_header_t *)(echo0+1);
 832       l4_header = ip4_next_header (inner_ip0);
 833
 834       if (!ip4_header_checksum_is_valid (inner_ip0))
 835         {
 836           next0 = SNAT_IN2OUT_NEXT_DROP;
 837           goto out;
 838         }
 839
 840       old_addr0 = inner_ip0->dst_address.as_u32;
 841       inner_ip0->dst_address = sm0.addr;
 842       new_addr0 = inner_ip0->dst_address.as_u32;
 843
 844       sum0 = icmp0->checksum;
 845       sum0 = ip_csum_update (sum0, old_addr0, new_addr0, ip4_header_t,
 846                              dst_address /* changed member */);
 847       icmp0->checksum = ip_csum_fold (sum0);
 848
 849       switch (protocol)
 850         {
 851           case SNAT_PROTOCOL_ICMP:
 852             inner_icmp0 = (icmp46_header_t*)l4_header;
 853             inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
 854
 855             old_id0 = inner_echo0->identifier;
 856             new_id0 = sm0.port;
 857             inner_echo0->identifier = new_id0;
 858
 859             sum0 = icmp0->checksum;
 860             sum0 = ip_csum_update (sum0, old_id0, new_id0, icmp_echo_header_t,
 861                                    identifier);
 862             icmp0->checksum = ip_csum_fold (sum0);
 863             break;
 864           case SNAT_PROTOCOL_UDP:
 865           case SNAT_PROTOCOL_TCP:
 866             old_id0 = ((tcp_udp_header_t*)l4_header)->dst_port;
 867             new_id0 = sm0.port;
 868             ((tcp_udp_header_t*)l4_header)->dst_port = new_id0;
 869
 870             sum0 = icmp0->checksum;
 871             sum0 = ip_csum_update (sum0, old_id0, new_id0, tcp_udp_header_t,
 872                                    dst_port);
 873             icmp0->checksum = ip_csum_fold (sum0);
 874             break;
 875           default:
 876             ASSERT(0);
 877         }
 878     }
 879
 880 out:
 881   return next0;
 882 }
 883
 884 /**
 885  * @brief Hairpinning
 886  *
 887  * Hairpinning allows two endpoints on the internal side of the NAT to
 888  * communicate even if they only use each other's external IP addresses
 889  * and ports.
 890  *
 891  * @param sm     NAT main.
 892  * @param b0     Vlib buffer.
 893  * @param ip0    IP header.
 894  * @param udp0   UDP header.
 895  * @param tcp0   TCP header.
 896  * @param proto0 NAT protocol.
 897  */
 898 static inline int
 899 snat_hairpinning (snat_main_t *sm,
 900                   vlib_buffer_t * b0,
 901                   ip4_header_t * ip0,
 902                   udp_header_t * udp0,
 903                   tcp_header_t * tcp0,
 904                   u32 proto0)
 905 {
 906   snat_session_key_t key0, sm0;
 907   snat_session_t * s0;
 908   clib_bihash_kv_8_8_t kv0, value0;
 909   ip_csum_t sum0;
 910   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
 911   u16 new_dst_port0, old_dst_port0;
 912
 913   key0.addr = ip0->dst_address;
 914   key0.port = udp0->dst_port;
 915   key0.protocol = proto0;
 916   key0.fib_index = sm->outside_fib_index;
 917   kv0.key = key0.as_u64;
 918
 919   /* Check if destination is static mappings */
 920   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
 921     {
 922       new_dst_addr0 = sm0.addr.as_u32;
 923       new_dst_port0 = sm0.port;
 924       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
 925     }
 926   /* or active session */
 927   else
 928     {
 929       if (sm->num_workers > 1)
 930         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
 931       else
 932         ti = sm->num_workers;
 933
 934       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
 935         {
 936           si = value0.value;
 937
 938           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
 939           new_dst_addr0 = s0->in2out.addr.as_u32;
 940           new_dst_port0 = s0->in2out.port;
 941           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
 942         }
 943     }
 944
 945   /* Destination is behind the same NAT, use internal address and port */
 946   if (new_dst_addr0)
 947     {
 948       old_dst_addr0 = ip0->dst_address.as_u32;
 949       ip0->dst_address.as_u32 = new_dst_addr0;
 950       sum0 = ip0->checksum;
 951       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 952                              ip4_header_t, dst_address);
 953       ip0->checksum = ip_csum_fold (sum0);
 954
 955       old_dst_port0 = tcp0->dst;
 956       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0))
 957         {
 958           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 959             {
 960               tcp0->dst = new_dst_port0;
 961               sum0 = tcp0->checksum;
 962               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 963                                      ip4_header_t, dst_address);
 964               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
 965                                      ip4_header_t /* cheat */, length);
 966               tcp0->checksum = ip_csum_fold(sum0);
 967             }
 968           else
 969             {
 970               udp0->dst_port = new_dst_port0;
 971               udp0->checksum = 0;
 972             }
 973         }
 974       else
 975         {
 976           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
 977             {
 978               sum0 = tcp0->checksum;
 979               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
 980                                      ip4_header_t, dst_address);
 981               tcp0->checksum = ip_csum_fold(sum0);
 982             }
 983         }
 984       return 1;
 985     }
 986   return 0;
 987 }
 988
 989 static inline void
 990 snat_icmp_hairpinning (snat_main_t *sm,
 991                        vlib_buffer_t * b0,
 992                        ip4_header_t * ip0,
 993                        icmp46_header_t * icmp0)
 994 {
 995   snat_session_key_t key0, sm0;
 996   clib_bihash_kv_8_8_t kv0, value0;
 997   u32 new_dst_addr0 = 0, old_dst_addr0, si, ti = 0;
 998   ip_csum_t sum0;
 999   snat_session_t *s0;
1000
1001   if (!icmp_is_error_message (icmp0))
1002     {
1003       icmp_echo_header_t *echo0 = (icmp_echo_header_t *)(icmp0+1);
1004       u16 icmp_id0 = echo0->identifier;
1005       key0.addr = ip0->dst_address;
1006       key0.port = icmp_id0;
1007       key0.protocol = SNAT_PROTOCOL_ICMP;
1008       key0.fib_index = sm->outside_fib_index;
1009       kv0.key = key0.as_u64;
1010
1011       if (sm->num_workers > 1)
1012         ti = (clib_net_to_host_u16 (icmp_id0) - 1024) / sm->port_per_thread;
1013       else
1014         ti = sm->num_workers;
1015
1016       /* Check if destination is in active sessions */
1017       if (clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0,
1018                                   &value0))
1019         {
1020           /* or static mappings */
1021           if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
1022             {
1023               new_dst_addr0 = sm0.addr.as_u32;
1024               vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
1025             }
1026         }
1027       else
1028         {
1029           si = value0.value;
1030
1031           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
1032           new_dst_addr0 = s0->in2out.addr.as_u32;
1033           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
1034           echo0->identifier = s0->in2out.port;
1035           sum0 = icmp0->checksum;
1036           sum0 = ip_csum_update (sum0, icmp_id0, s0->in2out.port,
1037                                  icmp_echo_header_t, identifier);
1038           icmp0->checksum = ip_csum_fold (sum0);
1039         }
1040
1041       /* Destination is behind the same NAT, use internal address and port */
1042       if (new_dst_addr0)
1043         {
1044           old_dst_addr0 = ip0->dst_address.as_u32;
1045           ip0->dst_address.as_u32 = new_dst_addr0;
1046           sum0 = ip0->checksum;
1047           sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
1048                                  ip4_header_t, dst_address);
1049           ip0->checksum = ip_csum_fold (sum0);
1050         }
1051     }
1052
1053 }
1054
1055 static inline u32 icmp_in2out_slow_path (snat_main_t *sm,
1056                                          vlib_buffer_t * b0,
1057                                          ip4_header_t * ip0,
1058                                          icmp46_header_t * icmp0,
1059                                          u32 sw_if_index0,
1060                                          u32 rx_fib_index0,
1061                                          vlib_node_runtime_t * node,
1062                                          u32 next0,
1063                                          f64 now,
1064                                          u32 thread_index,
1065                                          snat_session_t ** p_s0)
1066 {
1067   next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
1068                       next0, thread_index, p_s0, 0);
1069   snat_session_t * s0 = *p_s0;
1070   if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0))
1071     {
1072       /* Hairpinning */
1073       if (vnet_buffer(b0)->sw_if_index[VLIB_TX] == 0)
1074         snat_icmp_hairpinning(sm, b0, ip0, icmp0);
1075       /* Accounting */
1076       s0->last_heard = now;
1077       s0->total_pkts++;
1078       s0->total_bytes += vlib_buffer_length_in_chain (sm->vlib_main, b0);
1079       /* Per-user LRU list maintenance */
1080       clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1081                          s0->per_user_index);
1082       clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1083                           s0->per_user_list_head_index,
1084                           s0->per_user_index);
1085     }
1086   return next0;
1087 }
1088 static inline void
1089 snat_hairpinning_unknown_proto (snat_main_t *sm,
1090                                 vlib_buffer_t * b,
1091                                 ip4_header_t * ip)
1092 {
1093   u32 old_addr, new_addr = 0, ti = 0;
1094   clib_bihash_kv_8_8_t kv, value;
1095   clib_bihash_kv_16_8_t s_kv, s_value;
1096   nat_ed_ses_key_t key;
1097   snat_session_key_t m_key;
1098   snat_static_mapping_t *m;
1099   ip_csum_t sum;
1100   snat_session_t *s;
1101
1102   old_addr = ip->dst_address.as_u32;
1103   key.l_addr.as_u32 = ip->dst_address.as_u32;
1104   key.r_addr.as_u32 = ip->src_address.as_u32;
1105   key.fib_index = sm->outside_fib_index;
1106   key.proto = ip->protocol;
1107   key.r_port = 0;
1108   key.l_port = 0;
1109   s_kv.key[0] = key.as_u64[0];
1110   s_kv.key[1] = key.as_u64[1];
1111   if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1112     {
1113       m_key.addr = ip->dst_address;
1114       m_key.fib_index = sm->outside_fib_index;
1115       m_key.port = 0;
1116       m_key.protocol = 0;
1117       kv.key = m_key.as_u64;
1118       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
1119         return;
1120
1121       m = pool_elt_at_index (sm->static_mappings, value.value);
1122       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1123         vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
1124       new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
1125     }
1126   else
1127     {
1128       if (sm->num_workers > 1)
1129         ti = sm->worker_out2in_cb (ip, sm->outside_fib_index);
1130       else
1131         ti = sm->num_workers;
1132
1133       s = pool_elt_at_index (sm->per_thread_data[ti].sessions, s_value.value);
1134       if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1135         vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
1136       new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
1137     }
1138   sum = ip->checksum;
1139   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
1140   ip->checksum = ip_csum_fold (sum);
1141 }
1142
1143 static snat_session_t *
1144 snat_in2out_unknown_proto (snat_main_t *sm,
1145                            vlib_buffer_t * b,
1146                            ip4_header_t * ip,
1147                            u32 rx_fib_index,
1148                            u32 thread_index,
1149                            f64 now,
1150                            vlib_main_t * vm,
1151                            vlib_node_runtime_t * node)
1152 {
1153   clib_bihash_kv_8_8_t kv, value;
1154   clib_bihash_kv_16_8_t s_kv, s_value;
1155   snat_static_mapping_t *m;
1156   snat_session_key_t m_key;
1157   u32 old_addr, new_addr = 0;
1158   ip_csum_t sum;
1159   snat_user_t *u;
1160   dlist_elt_t *head, *elt;
1161   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1162   u32 elt_index, head_index, ses_index;
1163   snat_session_t * s;
1164   nat_ed_ses_key_t key;
1165   u32 address_index = ~0;
1166   int i;
1167   u8 is_sm = 0;
1168
1169   old_addr = ip->src_address.as_u32;
1170
1171   key.l_addr = ip->src_address;
1172   key.r_addr = ip->dst_address;
1173   key.fib_index = rx_fib_index;
1174   key.proto = ip->protocol;
1175   key.l_port = 0;
1176   key.r_port = 0;
1177   s_kv.key[0] = key.as_u64[0];
1178   s_kv.key[1] = key.as_u64[1];
1179
1180   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1181     {
1182       s = pool_elt_at_index (tsm->sessions, s_value.value);
1183       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1184     }
1185   else
1186     {
1187       if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
1188         {
1189           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1190           nat_ipfix_logging_max_sessions(sm->max_translations);
1191           return 0;
1192         }
1193
1194       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1195                                   thread_index);
1196       if (!u)
1197         {
1198           clib_warning ("create NAT user failed");
1199           return 0;
1200         }
1201
1202       m_key.addr = ip->src_address;
1203       m_key.port = 0;
1204       m_key.protocol = 0;
1205       m_key.fib_index = rx_fib_index;
1206       kv.key = m_key.as_u64;
1207
1208       /* Try to find static mapping first */
1209       if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
1210         {
1211           m = pool_elt_at_index (sm->static_mappings, value.value);
1212           new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
1213           is_sm = 1;
1214           goto create_ses;
1215         }
1216       /* Fallback to 3-tuple key */
1217       else
1218         {
1219           /* Choose same out address as for TCP/UDP session to same destination */
1220           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
1221             {
1222               head_index = u->sessions_per_user_list_head_index;
1223               head = pool_elt_at_index (tsm->list_pool, head_index);
1224               elt_index = head->next;
1225               elt = pool_elt_at_index (tsm->list_pool, elt_index);
1226               ses_index = elt->value;
1227               while (ses_index != ~0)
1228                 {
1229                   s =  pool_elt_at_index (tsm->sessions, ses_index);
1230                   elt_index = elt->next;
1231                   elt = pool_elt_at_index (tsm->list_pool, elt_index);
1232                   ses_index = elt->value;
1233
1234                   if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
1235                     {
1236                       new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1237                       address_index = s->outside_address_index;
1238
1239                       key.fib_index = sm->outside_fib_index;
1240                       key.l_addr.as_u32 = new_addr;
1241                       s_kv.key[0] = key.as_u64[0];
1242                       s_kv.key[1] = key.as_u64[1];
1243                       if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1244                         break;
1245
1246                       goto create_ses;
1247                     }
1248                 }
1249             }
1250           key.fib_index = sm->outside_fib_index;
1251           for (i = 0; i < vec_len (sm->addresses); i++)
1252             {
1253               key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
1254               s_kv.key[0] = key.as_u64[0];
1255               s_kv.key[1] = key.as_u64[1];
1256               if (clib_bihash_search_16_8 (&sm->out2in_ed, &s_kv, &s_value))
1257                 {
1258                   new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
1259                   address_index = i;
1260                   goto create_ses;
1261                 }
1262             }
1263           return 0;
1264         }
1265
1266 create_ses:
1267       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1268       if (!s)
1269         {
1270           clib_warning ("create NAT session failed");
1271           return 0;
1272         }
1273
1274       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1275       s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
1276       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1277       s->outside_address_index = address_index;
1278       s->out2in.addr.as_u32 = new_addr;
1279       s->out2in.fib_index = sm->outside_fib_index;
1280       s->in2out.addr.as_u32 = old_addr;
1281       s->in2out.fib_index = rx_fib_index;
1282       s->in2out.port = s->out2in.port = ip->protocol;
1283       if (is_sm)
1284         s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1285       user_session_increment (sm, u, is_sm);
1286
1287       /* Add to lookup tables */
1288       key.l_addr.as_u32 = old_addr;
1289       key.r_addr = ip->dst_address;
1290       key.proto = ip->protocol;
1291       key.fib_index = rx_fib_index;
1292       s_kv.key[0] = key.as_u64[0];
1293       s_kv.key[1] = key.as_u64[1];
1294       s_kv.value = s - tsm->sessions;
1295       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1296         clib_warning ("in2out key add failed");
1297
1298       key.l_addr.as_u32 = new_addr;
1299       key.fib_index = sm->outside_fib_index;
1300       s_kv.key[0] = key.as_u64[0];
1301       s_kv.key[1] = key.as_u64[1];
1302       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1303         clib_warning ("out2in key add failed");
1304   }
1305
1306   /* Update IP checksum */
1307   sum = ip->checksum;
1308   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1309   ip->checksum = ip_csum_fold (sum);
1310
1311   /* Accounting */
1312   s->last_heard = now;
1313   s->total_pkts++;
1314   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1315   /* Per-user LRU list maintenance */
1316   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1317   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1318                       s->per_user_index);
1319
1320   /* Hairpinning */
1321   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1322     snat_hairpinning_unknown_proto(sm, b, ip);
1323
1324   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1325     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1326
1327   return s;
1328 }
1329
1330 static snat_session_t *
1331 snat_in2out_lb (snat_main_t *sm,
1332                 vlib_buffer_t * b,
1333                 ip4_header_t * ip,
1334                 u32 rx_fib_index,
1335                 u32 thread_index,
1336                 f64 now,
1337                 vlib_main_t * vm,
1338                 vlib_node_runtime_t * node)
1339 {
1340   nat_ed_ses_key_t key;
1341   clib_bihash_kv_16_8_t s_kv, s_value;
1342   udp_header_t *udp = ip4_next_header (ip);
1343   tcp_header_t *tcp = (tcp_header_t *) udp;
1344   snat_session_t *s = 0;
1345   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
1346   u32 old_addr, new_addr;
1347   u16 new_port, old_port;
1348   ip_csum_t sum;
1349   u32 proto = ip_proto_to_snat_proto (ip->protocol);
1350   snat_session_key_t e_key, l_key;
1351   snat_user_t *u;
1352   u8 lb;
1353
1354   old_addr = ip->src_address.as_u32;
1355
1356   key.l_addr = ip->src_address;
1357   key.r_addr = ip->dst_address;
1358   key.fib_index = rx_fib_index;
1359   key.proto = ip->protocol;
1360   key.r_port = udp->dst_port;
1361   key.l_port = udp->src_port;
1362   s_kv.key[0] = key.as_u64[0];
1363   s_kv.key[1] = key.as_u64[1];
1364
1365   if (!clib_bihash_search_16_8 (&sm->in2out_ed, &s_kv, &s_value))
1366     {
1367       s = pool_elt_at_index (tsm->sessions, s_value.value);
1368       if (is_fwd_bypass_session (s))
1369         return 0;
1370     }
1371   else
1372     {
1373       if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
1374         {
1375           b->error = node->errors[SNAT_IN2OUT_ERROR_MAX_SESSIONS_EXCEEDED];
1376           nat_ipfix_logging_max_sessions(sm->max_translations);
1377           return 0;
1378         }
1379
1380       l_key.addr = ip->src_address;
1381       l_key.port = udp->src_port;
1382       l_key.protocol = proto;
1383       l_key.fib_index = rx_fib_index;
1384       if (snat_static_mapping_match(sm, l_key, &e_key, 0, 0, 0, &lb))
1385         return 0;
1386
1387       u = nat_user_get_or_create (sm, &ip->src_address, rx_fib_index,
1388                                   thread_index);
1389       if (!u)
1390         {
1391           clib_warning ("create NAT user failed");
1392           return 0;
1393         }
1394
1395       s = nat_session_alloc_or_recycle (sm, u, thread_index);
1396       if (!s)
1397         {
1398           clib_warning ("create NAT session failed");
1399           return 0;
1400         }
1401
1402       s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
1403       s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
1404       if (lb)
1405         s->flags |= SNAT_SESSION_FLAG_LOAD_BALANCING;
1406       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
1407       s->outside_address_index = ~0;
1408       s->in2out = l_key;
1409       s->out2in = e_key;
1410       s->out2in.protocol = l_key.protocol;
1411       user_session_increment (sm, u, 1 /* static */);
1412
1413       /* Add to lookup tables */
1414       s_kv.value = s - tsm->sessions;
1415       if (clib_bihash_add_del_16_8 (&sm->in2out_ed, &s_kv, 1))
1416         clib_warning ("in2out-ed key add failed");
1417
1418       key.l_addr = e_key.addr;
1419       key.fib_index = e_key.fib_index;
1420       key.l_port = e_key.port;
1421       s_kv.key[0] = key.as_u64[0];
1422       s_kv.key[1] = key.as_u64[1];
1423       if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &s_kv, 1))
1424         clib_warning ("out2in-ed key add failed");
1425     }
1426
1427   new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
1428
1429   /* Update IP checksum */
1430   sum = ip->checksum;
1431   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1432   if (is_twice_nat_session (s))
1433     sum = ip_csum_update (sum, ip->dst_address.as_u32,
1434                           s->ext_host_addr.as_u32, ip4_header_t, dst_address);
1435   ip->checksum = ip_csum_fold (sum);
1436
1437   if (PREDICT_TRUE(proto == SNAT_PROTOCOL_TCP))
1438     {
1439       old_port = tcp->src_port;
1440       tcp->src_port = s->out2in.port;
1441       new_port = tcp->src_port;
1442
1443       sum = tcp->checksum;
1444       sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
1445       sum = ip_csum_update (sum, old_port, new_port, ip4_header_t, length);
1446       if (is_twice_nat_session (s))
1447         {
1448           sum = ip_csum_update (sum, ip->dst_address.as_u32,
1449                                 s->ext_host_addr.as_u32, ip4_header_t,
1450                                 dst_address);
1451           sum = ip_csum_update (sum, tcp->dst_port, s->ext_host_port,
1452                                 ip4_header_t, length);
1453           tcp->dst_port = s->ext_host_port;
1454           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1455         }
1456       tcp->checksum = ip_csum_fold(sum);
1457     }
1458   else
1459     {
1460       udp->src_port = s->out2in.port;
1461       if (is_twice_nat_session (s))
1462         {
1463           udp->dst_port = s->ext_host_port;
1464           ip->dst_address.as_u32 = s->ext_host_addr.as_u32;
1465         }
1466       udp->checksum = 0;
1467     }
1468
1469   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
1470     vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
1471
1472   /* Accounting */
1473   s->last_heard = now;
1474   s->total_pkts++;
1475   s->total_bytes += vlib_buffer_length_in_chain (vm, b);
1476   /* Per-user LRU list maintenance */
1477   clib_dlist_remove (tsm->list_pool, s->per_user_index);
1478   clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
1479                       s->per_user_index);
1480   return s;
1481 }
1482
1483 static inline uword
1484 snat_in2out_node_fn_inline (vlib_main_t * vm,
1485                             vlib_node_runtime_t * node,
1486                             vlib_frame_t * frame, int is_slow_path,
1487                             int is_output_feature)
1488 {
1489   u32 n_left_from, * from, * to_next;
1490   snat_in2out_next_t next_index;
1491   u32 pkts_processed = 0;
1492   snat_main_t * sm = &snat_main;
1493   f64 now = vlib_time_now (vm);
1494   u32 stats_node_index;
1495   u32 thread_index = vlib_get_thread_index ();
1496
1497   stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index :
1498     snat_in2out_node.index;
1499
1500   from = vlib_frame_vector_args (frame);
1501   n_left_from = frame->n_vectors;
1502   next_index = node->cached_next_index;
1503
1504   while (n_left_from > 0)
1505     {
1506       u32 n_left_to_next;
1507
1508       vlib_get_next_frame (vm, node, next_index,
1509                            to_next, n_left_to_next);
1510
1511       while (n_left_from >= 4 && n_left_to_next >= 2)
1512         {
1513           u32 bi0, bi1;
1514           vlib_buffer_t * b0, * b1;
1515           u32 next0, next1;
1516           u32 sw_if_index0, sw_if_index1;
1517           ip4_header_t * ip0, * ip1;
1518           ip_csum_t sum0, sum1;
1519           u32 new_addr0, old_addr0, new_addr1, old_addr1;
1520           u16 old_port0, new_port0, old_port1, new_port1;
1521           udp_header_t * udp0, * udp1;
1522           tcp_header_t * tcp0, * tcp1;
1523           icmp46_header_t * icmp0, * icmp1;
1524           snat_session_key_t key0, key1;
1525           u32 rx_fib_index0, rx_fib_index1;
1526           u32 proto0, proto1;
1527           snat_session_t * s0 = 0, * s1 = 0;
1528           clib_bihash_kv_8_8_t kv0, value0, kv1, value1;
1529           u32 iph_offset0 = 0, iph_offset1 = 0;
1530
1531           /* Prefetch next iteration. */
1532           {
1533             vlib_buffer_t * p2, * p3;
1534
1535             p2 = vlib_get_buffer (vm, from[2]);
1536             p3 = vlib_get_buffer (vm, from[3]);
1537
1538             vlib_prefetch_buffer_header (p2, LOAD);
1539             vlib_prefetch_buffer_header (p3, LOAD);
1540
1541             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
1542             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
1543           }
1544
1545           /* speculatively enqueue b0 and b1 to the current next frame */
1546           to_next[0] = bi0 = from[0];
1547           to_next[1] = bi1 = from[1];
1548           from += 2;
1549           to_next += 2;
1550           n_left_from -= 2;
1551           n_left_to_next -= 2;
1552
1553           b0 = vlib_get_buffer (vm, bi0);
1554           b1 = vlib_get_buffer (vm, bi1);
1555
1556           if (is_output_feature)
1557             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1558
1559           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1560                  iph_offset0);
1561
1562           udp0 = ip4_next_header (ip0);
1563           tcp0 = (tcp_header_t *) udp0;
1564           icmp0 = (icmp46_header_t *) udp0;
1565
1566           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1567           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1568                                    sw_if_index0);
1569
1570           next0 = next1 = SNAT_IN2OUT_NEXT_LOOKUP;
1571
1572           if (PREDICT_FALSE(ip0->ttl == 1))
1573             {
1574               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1575               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1576                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1577                                            0);
1578               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1579               goto trace00;
1580             }
1581
1582           proto0 = ip_proto_to_snat_proto (ip0->protocol);
1583
1584           /* Next configured feature, probably ip4-lookup */
1585           if (is_slow_path)
1586             {
1587               if (PREDICT_FALSE (proto0 == ~0))
1588                 {
1589                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
1590                                                   thread_index, now, vm, node);
1591                   if (!s0)
1592                     next0 = SNAT_IN2OUT_NEXT_DROP;
1593                   goto trace00;
1594                 }
1595
1596               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
1597                 {
1598                   next0 = icmp_in2out_slow_path
1599                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0,
1600                      node, next0, now, thread_index, &s0);
1601                   goto trace00;
1602                 }
1603             }
1604           else
1605             {
1606               if (is_output_feature)
1607                 {
1608                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
1609                     goto trace00;
1610                 }
1611
1612               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
1613                 {
1614                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1615                   goto trace00;
1616                 }
1617
1618               if (ip4_is_fragment (ip0))
1619                 {
1620                   next0 = SNAT_IN2OUT_NEXT_REASS;
1621                   goto trace00;
1622                 }
1623             }
1624
1625           key0.addr = ip0->src_address;
1626           key0.port = udp0->src_port;
1627           key0.protocol = proto0;
1628           key0.fib_index = rx_fib_index0;
1629
1630           kv0.key = key0.as_u64;
1631
1632           if (PREDICT_FALSE (clib_bihash_search_8_8 (
1633               &sm->per_thread_data[thread_index].in2out, &kv0, &value0) != 0))
1634             {
1635               if (is_slow_path)
1636                 {
1637                   if (is_output_feature)
1638                     {
1639                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1640                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
1641                         goto trace00;
1642                     }
1643                   else
1644                     {
1645                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
1646                           ip0, proto0, rx_fib_index0, thread_index)))
1647                         goto trace00;
1648                     }
1649
1650                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
1651                                      &s0, node, next0, thread_index);
1652                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
1653                     goto trace00;
1654                 }
1655               else
1656                 {
1657                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1658                   goto trace00;
1659                 }
1660             }
1661           else
1662             {
1663               if (PREDICT_FALSE (value0.value == ~0ULL))
1664                 {
1665                   if (is_slow_path)
1666                     {
1667                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
1668                                           thread_index, now, vm, node);
1669                       if (!s0 && !sm->forwarding_enabled)
1670                         next0 = SNAT_IN2OUT_NEXT_DROP;
1671                       goto trace00;
1672                     }
1673                   else
1674                     {
1675                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1676                       goto trace00;
1677                     }
1678                 }
1679               else
1680                 {
1681                   s0 = pool_elt_at_index (
1682                     sm->per_thread_data[thread_index].sessions,
1683                     value0.value);
1684                 }
1685             }
1686
1687           b0->flags |= VNET_BUFFER_F_IS_NATED;
1688
1689           old_addr0 = ip0->src_address.as_u32;
1690           ip0->src_address = s0->out2in.addr;
1691           new_addr0 = ip0->src_address.as_u32;
1692           if (!is_output_feature)
1693             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
1694
1695           sum0 = ip0->checksum;
1696           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1697                                  ip4_header_t,
1698                                  src_address /* changed member */);
1699           ip0->checksum = ip_csum_fold (sum0);
1700
1701           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
1702             {
1703               old_port0 = tcp0->src_port;
1704               tcp0->src_port = s0->out2in.port;
1705               new_port0 = tcp0->src_port;
1706
1707               sum0 = tcp0->checksum;
1708               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
1709                                      ip4_header_t,
1710                                      dst_address /* changed member */);
1711               sum0 = ip_csum_update (sum0, old_port0, new_port0,
1712                                      ip4_header_t /* cheat */,
1713                                      length /* changed member */);
1714               tcp0->checksum = ip_csum_fold(sum0);
1715             }
1716           else
1717             {
1718               old_port0 = udp0->src_port;
1719               udp0->src_port = s0->out2in.port;
1720               udp0->checksum = 0;
1721             }
1722
1723           /* Accounting */
1724           s0->last_heard = now;
1725           s0->total_pkts++;
1726           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
1727           /* Per-user LRU list maintenance */
1728           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1729                              s0->per_user_index);
1730           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1731                               s0->per_user_list_head_index,
1732                               s0->per_user_index);
1733         trace00:
1734
1735           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1736                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
1737             {
1738               snat_in2out_trace_t *t =
1739                  vlib_add_trace (vm, node, b0, sizeof (*t));
1740               t->is_slow_path = is_slow_path;
1741               t->sw_if_index = sw_if_index0;
1742               t->next_index = next0;
1743                   t->session_index = ~0;
1744               if (s0)
1745                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
1746             }
1747
1748           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
1749
1750           if (is_output_feature)
1751             iph_offset1 = vnet_buffer (b1)->ip.save_rewrite_length;
1752
1753           ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) +
1754                  iph_offset1);
1755
1756           udp1 = ip4_next_header (ip1);
1757           tcp1 = (tcp_header_t *) udp1;
1758           icmp1 = (icmp46_header_t *) udp1;
1759
1760           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
1761           rx_fib_index1 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1762                                    sw_if_index1);
1763
1764           if (PREDICT_FALSE(ip1->ttl == 1))
1765             {
1766               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1767               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
1768                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1769                                            0);
1770               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1771               goto trace01;
1772             }
1773
1774           proto1 = ip_proto_to_snat_proto (ip1->protocol);
1775
1776           /* Next configured feature, probably ip4-lookup */
1777           if (is_slow_path)
1778             {
1779               if (PREDICT_FALSE (proto1 == ~0))
1780                 {
1781                   s1 = snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
1782                                                   thread_index, now, vm, node);
1783                   if (!s1)
1784                     next1 = SNAT_IN2OUT_NEXT_DROP;
1785                   goto trace01;
1786                 }
1787
1788               if (PREDICT_FALSE (proto1 == SNAT_PROTOCOL_ICMP))
1789                 {
1790                   next1 = icmp_in2out_slow_path
1791                     (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node,
1792                      next1, now, thread_index, &s1);
1793                   goto trace01;
1794                 }
1795             }
1796           else
1797             {
1798               if (is_output_feature)
1799                 {
1800                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip1, thread_index)))
1801                     goto trace01;
1802                 }
1803
1804               if (PREDICT_FALSE (proto1 == ~0 || proto1 == SNAT_PROTOCOL_ICMP))
1805                 {
1806                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1807                   goto trace01;
1808                 }
1809
1810               if (ip4_is_fragment (ip1))
1811                 {
1812                   next1 = SNAT_IN2OUT_NEXT_REASS;
1813                   goto trace01;
1814                 }
1815             }
1816
1817           key1.addr = ip1->src_address;
1818           key1.port = udp1->src_port;
1819           key1.protocol = proto1;
1820           key1.fib_index = rx_fib_index1;
1821
1822           kv1.key = key1.as_u64;
1823
1824             if (PREDICT_FALSE(clib_bihash_search_8_8 (
1825                 &sm->per_thread_data[thread_index].in2out, &kv1, &value1) != 0))
1826             {
1827               if (is_slow_path)
1828                 {
1829                   if (is_output_feature)
1830                     {
1831                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
1832                           ip1, proto1, udp1->src_port, udp1->dst_port, thread_index, sw_if_index1)))
1833                         goto trace01;
1834                     }
1835                   else
1836                     {
1837                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index1,
1838                           ip1, proto1, rx_fib_index1, thread_index)))
1839                         goto trace01;
1840                     }
1841
1842                   next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1,
1843                                      &s1, node, next1, thread_index);
1844                   if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP))
1845                     goto trace01;
1846                 }
1847               else
1848                 {
1849                   next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1850                   goto trace01;
1851                 }
1852             }
1853           else
1854             {
1855               if (PREDICT_FALSE (value1.value == ~0ULL))
1856                 {
1857                   if (is_slow_path)
1858                     {
1859                       s1 = snat_in2out_lb(sm, b1, ip1, rx_fib_index1,
1860                                           thread_index, now, vm, node);
1861                       if (!s1 && !sm->forwarding_enabled)
1862                         next1 = SNAT_IN2OUT_NEXT_DROP;
1863                       goto trace01;
1864                     }
1865                   else
1866                     {
1867                       next1 = SNAT_IN2OUT_NEXT_SLOW_PATH;
1868                       goto trace01;
1869                     }
1870                 }
1871               else
1872                 {
1873                   s1 = pool_elt_at_index (
1874                     sm->per_thread_data[thread_index].sessions,
1875                     value1.value);
1876                 }
1877             }
1878
1879           b1->flags |= VNET_BUFFER_F_IS_NATED;
1880
1881           old_addr1 = ip1->src_address.as_u32;
1882           ip1->src_address = s1->out2in.addr;
1883           new_addr1 = ip1->src_address.as_u32;
1884           if (!is_output_feature)
1885             vnet_buffer(b1)->sw_if_index[VLIB_TX] = s1->out2in.fib_index;
1886
1887           sum1 = ip1->checksum;
1888           sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1889                                  ip4_header_t,
1890                                  src_address /* changed member */);
1891           ip1->checksum = ip_csum_fold (sum1);
1892
1893           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
1894             {
1895               old_port1 = tcp1->src_port;
1896               tcp1->src_port = s1->out2in.port;
1897               new_port1 = tcp1->src_port;
1898
1899               sum1 = tcp1->checksum;
1900               sum1 = ip_csum_update (sum1, old_addr1, new_addr1,
1901                                      ip4_header_t,
1902                                      dst_address /* changed member */);
1903               sum1 = ip_csum_update (sum1, old_port1, new_port1,
1904                                      ip4_header_t /* cheat */,
1905                                      length /* changed member */);
1906               tcp1->checksum = ip_csum_fold(sum1);
1907             }
1908           else
1909             {
1910               old_port1 = udp1->src_port;
1911               udp1->src_port = s1->out2in.port;
1912               udp1->checksum = 0;
1913             }
1914
1915           /* Accounting */
1916           s1->last_heard = now;
1917           s1->total_pkts++;
1918           s1->total_bytes += vlib_buffer_length_in_chain (vm, b1);
1919           /* Per-user LRU list maintenance */
1920           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
1921                              s1->per_user_index);
1922           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
1923                               s1->per_user_list_head_index,
1924                               s1->per_user_index);
1925         trace01:
1926
1927           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
1928                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
1929             {
1930               snat_in2out_trace_t *t =
1931                  vlib_add_trace (vm, node, b1, sizeof (*t));
1932               t->sw_if_index = sw_if_index1;
1933               t->next_index = next1;
1934               t->session_index = ~0;
1935               if (s1)
1936                 t->session_index = s1 - sm->per_thread_data[thread_index].sessions;
1937             }
1938
1939           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
1940
1941           /* verify speculative enqueues, maybe switch current next frame */
1942           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1943                                            to_next, n_left_to_next,
1944                                            bi0, bi1, next0, next1);
1945         }
1946
1947       while (n_left_from > 0 && n_left_to_next > 0)
1948         {
1949           u32 bi0;
1950           vlib_buffer_t * b0;
1951           u32 next0;
1952           u32 sw_if_index0;
1953           ip4_header_t * ip0;
1954           ip_csum_t sum0;
1955           u32 new_addr0, old_addr0;
1956           u16 old_port0, new_port0;
1957           udp_header_t * udp0;
1958           tcp_header_t * tcp0;
1959           icmp46_header_t * icmp0;
1960           snat_session_key_t key0;
1961           u32 rx_fib_index0;
1962           u32 proto0;
1963           snat_session_t * s0 = 0;
1964           clib_bihash_kv_8_8_t kv0, value0;
1965           u32 iph_offset0 = 0;
1966
1967           /* speculatively enqueue b0 to the current next frame */
1968           bi0 = from[0];
1969           to_next[0] = bi0;
1970           from += 1;
1971           to_next += 1;
1972           n_left_from -= 1;
1973           n_left_to_next -= 1;
1974
1975           b0 = vlib_get_buffer (vm, bi0);
1976           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
1977
1978           if (is_output_feature)
1979             iph_offset0 = vnet_buffer (b0)->ip.save_rewrite_length;
1980
1981           ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) +
1982                  iph_offset0);
1983
1984           udp0 = ip4_next_header (ip0);
1985           tcp0 = (tcp_header_t *) udp0;
1986           icmp0 = (icmp46_header_t *) udp0;
1987
1988           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
1989           rx_fib_index0 = vec_elt (sm->ip4_main->fib_index_by_sw_if_index,
1990                                    sw_if_index0);
1991
1992           if (PREDICT_FALSE(ip0->ttl == 1))
1993             {
1994               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1995               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
1996                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
1997                                            0);
1998               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
1999               goto trace0;
2000             }
2001
2002           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2003
2004           /* Next configured feature, probably ip4-lookup */
2005           if (is_slow_path)
2006             {
2007               if (PREDICT_FALSE (proto0 == ~0))
2008                 {
2009                   s0 = snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
2010                                                   thread_index, now, vm, node);
2011                   if (!s0)
2012                     next0 = SNAT_IN2OUT_NEXT_DROP;
2013                   goto trace0;
2014                 }
2015
2016               if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
2017                 {
2018                   next0 = icmp_in2out_slow_path
2019                     (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node,
2020                      next0, now, thread_index, &s0);
2021                   goto trace0;
2022                 }
2023             }
2024           else
2025             {
2026                if (is_output_feature)
2027                 {
2028                   if (PREDICT_FALSE(nat_not_translate_output_feature_fwd(sm, ip0, thread_index)))
2029                     goto trace0;
2030                 }
2031
2032               if (PREDICT_FALSE (proto0 == ~0 || proto0 == SNAT_PROTOCOL_ICMP))
2033                 {
2034                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2035                   goto trace0;
2036                 }
2037
2038               if (ip4_is_fragment (ip0))
2039                 {
2040                   next0 = SNAT_IN2OUT_NEXT_REASS;
2041                   goto trace0;
2042                 }
2043             }
2044
2045           key0.addr = ip0->src_address;
2046           key0.port = udp0->src_port;
2047           key0.protocol = proto0;
2048           key0.fib_index = rx_fib_index0;
2049
2050           kv0.key = key0.as_u64;
2051
2052           if (clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out,
2053                                       &kv0, &value0))
2054             {
2055               if (is_slow_path)
2056                 {
2057                   if (is_output_feature)
2058                     {
2059                       if (PREDICT_FALSE(nat_not_translate_output_feature(sm,
2060                           ip0, proto0, udp0->src_port, udp0->dst_port, thread_index, sw_if_index0)))
2061                         goto trace0;
2062                     }
2063                   else
2064                     {
2065                       if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2066                           ip0, proto0, rx_fib_index0, thread_index)))
2067                         goto trace0;
2068                     }
2069
2070                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2071                                      &s0, node, next0, thread_index);
2072
2073                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2074                     goto trace0;
2075                 }
2076               else
2077                 {
2078                   next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2079                   goto trace0;
2080                 }
2081             }
2082           else
2083             {
2084               if (PREDICT_FALSE (value0.value == ~0ULL))
2085                 {
2086                   if (is_slow_path)
2087                     {
2088                       s0 = snat_in2out_lb(sm, b0, ip0, rx_fib_index0,
2089                                           thread_index, now, vm, node);
2090                       if (!s0 && !sm->forwarding_enabled)
2091                         next0 = SNAT_IN2OUT_NEXT_DROP;
2092                       goto trace0;
2093                     }
2094                   else
2095                     {
2096                       next0 = SNAT_IN2OUT_NEXT_SLOW_PATH;
2097                       goto trace0;
2098                     }
2099                 }
2100               else
2101                 {
2102                   s0 = pool_elt_at_index (
2103                     sm->per_thread_data[thread_index].sessions,
2104                     value0.value);
2105                 }
2106             }
2107
2108           b0->flags |= VNET_BUFFER_F_IS_NATED;
2109
2110           old_addr0 = ip0->src_address.as_u32;
2111           ip0->src_address = s0->out2in.addr;
2112           new_addr0 = ip0->src_address.as_u32;
2113           if (!is_output_feature)
2114             vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2115
2116           sum0 = ip0->checksum;
2117           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2118                                  ip4_header_t,
2119                                  src_address /* changed member */);
2120           ip0->checksum = ip_csum_fold (sum0);
2121
2122           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2123             {
2124               old_port0 = tcp0->src_port;
2125               tcp0->src_port = s0->out2in.port;
2126               new_port0 = tcp0->src_port;
2127
2128               sum0 = tcp0->checksum;
2129               sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2130                                      ip4_header_t,
2131                                      dst_address /* changed member */);
2132               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2133                                      ip4_header_t /* cheat */,
2134                                      length /* changed member */);
2135               tcp0->checksum = ip_csum_fold(sum0);
2136             }
2137           else
2138             {
2139               old_port0 = udp0->src_port;
2140               udp0->src_port = s0->out2in.port;
2141               udp0->checksum = 0;
2142             }
2143
2144           /* Accounting */
2145           s0->last_heard = now;
2146           s0->total_pkts++;
2147           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2148           /* Per-user LRU list maintenance */
2149           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2150                              s0->per_user_index);
2151           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2152                               s0->per_user_list_head_index,
2153                               s0->per_user_index);
2154
2155         trace0:
2156           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2157                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2158             {
2159               snat_in2out_trace_t *t =
2160                  vlib_add_trace (vm, node, b0, sizeof (*t));
2161               t->is_slow_path = is_slow_path;
2162               t->sw_if_index = sw_if_index0;
2163               t->next_index = next0;
2164                   t->session_index = ~0;
2165               if (s0)
2166                 t->session_index = s0 - sm->per_thread_data[thread_index].sessions;
2167             }
2168
2169           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2170
2171           /* verify speculative enqueue, maybe switch current next frame */
2172           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2173                                            to_next, n_left_to_next,
2174                                            bi0, next0);
2175         }
2176
2177       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2178     }
2179
2180   vlib_node_increment_counter (vm, stats_node_index,
2181                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2182                                pkts_processed);
2183   return frame->n_vectors;
2184 }
2185
2186 static uword
2187 snat_in2out_fast_path_fn (vlib_main_t * vm,
2188                           vlib_node_runtime_t * node,
2189                           vlib_frame_t * frame)
2190 {
2191   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 0);
2192 }
2193
2194 VLIB_REGISTER_NODE (snat_in2out_node) = {
2195   .function = snat_in2out_fast_path_fn,
2196   .name = "nat44-in2out",
2197   .vector_size = sizeof (u32),
2198   .format_trace = format_snat_in2out_trace,
2199   .type = VLIB_NODE_TYPE_INTERNAL,
2200
2201   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2202   .error_strings = snat_in2out_error_strings,
2203
2204   .runtime_data_bytes = sizeof (snat_runtime_t),
2205
2206   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2207
2208   /* edit / add dispositions here */
2209   .next_nodes = {
2210     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2211     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2212     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2213     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2214     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2215   },
2216 };
2217
2218 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_node, snat_in2out_fast_path_fn);
2219
2220 static uword
2221 snat_in2out_output_fast_path_fn (vlib_main_t * vm,
2222                                  vlib_node_runtime_t * node,
2223                                  vlib_frame_t * frame)
2224 {
2225   return snat_in2out_node_fn_inline (vm, node, frame, 0 /* is_slow_path */, 1);
2226 }
2227
2228 VLIB_REGISTER_NODE (snat_in2out_output_node) = {
2229   .function = snat_in2out_output_fast_path_fn,
2230   .name = "nat44-in2out-output",
2231   .vector_size = sizeof (u32),
2232   .format_trace = format_snat_in2out_trace,
2233   .type = VLIB_NODE_TYPE_INTERNAL,
2234
2235   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2236   .error_strings = snat_in2out_error_strings,
2237
2238   .runtime_data_bytes = sizeof (snat_runtime_t),
2239
2240   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2241
2242   /* edit / add dispositions here */
2243   .next_nodes = {
2244     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2245     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2246     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2247     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2248     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2249   },
2250 };
2251
2252 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_node,
2253                               snat_in2out_output_fast_path_fn);
2254
2255 static uword
2256 snat_in2out_slow_path_fn (vlib_main_t * vm,
2257                           vlib_node_runtime_t * node,
2258                           vlib_frame_t * frame)
2259 {
2260   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 0);
2261 }
2262
2263 VLIB_REGISTER_NODE (snat_in2out_slowpath_node) = {
2264   .function = snat_in2out_slow_path_fn,
2265   .name = "nat44-in2out-slowpath",
2266   .vector_size = sizeof (u32),
2267   .format_trace = format_snat_in2out_trace,
2268   .type = VLIB_NODE_TYPE_INTERNAL,
2269
2270   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2271   .error_strings = snat_in2out_error_strings,
2272
2273   .runtime_data_bytes = sizeof (snat_runtime_t),
2274
2275   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2276
2277   /* edit / add dispositions here */
2278   .next_nodes = {
2279     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2280     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2281     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2282     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2283     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2284   },
2285 };
2286
2287 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_slowpath_node,
2288                               snat_in2out_slow_path_fn);
2289
2290 static uword
2291 snat_in2out_output_slow_path_fn (vlib_main_t * vm,
2292                                  vlib_node_runtime_t * node,
2293                                  vlib_frame_t * frame)
2294 {
2295   return snat_in2out_node_fn_inline (vm, node, frame, 1 /* is_slow_path */, 1);
2296 }
2297
2298 VLIB_REGISTER_NODE (snat_in2out_output_slowpath_node) = {
2299   .function = snat_in2out_output_slow_path_fn,
2300   .name = "nat44-in2out-output-slowpath",
2301   .vector_size = sizeof (u32),
2302   .format_trace = format_snat_in2out_trace,
2303   .type = VLIB_NODE_TYPE_INTERNAL,
2304
2305   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2306   .error_strings = snat_in2out_error_strings,
2307
2308   .runtime_data_bytes = sizeof (snat_runtime_t),
2309
2310   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2311
2312   /* edit / add dispositions here */
2313   .next_nodes = {
2314     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2315     [SNAT_IN2OUT_NEXT_LOOKUP] = "interface-output",
2316     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-output-slowpath",
2317     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2318     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2319   },
2320 };
2321
2322 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_slowpath_node,
2323                               snat_in2out_output_slow_path_fn);
2324
2325 extern vnet_feature_arc_registration_t vnet_feat_arc_ip4_local;
2326
2327 static uword
2328 nat44_hairpinning_fn (vlib_main_t * vm,
2329                       vlib_node_runtime_t * node,
2330                       vlib_frame_t * frame)
2331 {
2332   u32 n_left_from, * from, * to_next;
2333   snat_in2out_next_t next_index;
2334   u32 pkts_processed = 0;
2335   snat_main_t * sm = &snat_main;
2336   vnet_feature_main_t *fm = &feature_main;
2337   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
2338   vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc_index];
2339
2340   from = vlib_frame_vector_args (frame);
2341   n_left_from = frame->n_vectors;
2342   next_index = node->cached_next_index;
2343
2344   while (n_left_from > 0)
2345     {
2346       u32 n_left_to_next;
2347
2348       vlib_get_next_frame (vm, node, next_index,
2349                            to_next, n_left_to_next);
2350
2351       while (n_left_from > 0 && n_left_to_next > 0)
2352         {
2353           u32 bi0;
2354           vlib_buffer_t * b0;
2355           u32 next0;
2356           ip4_header_t * ip0;
2357           u32 proto0;
2358           udp_header_t * udp0;
2359           tcp_header_t * tcp0;
2360
2361           /* speculatively enqueue b0 to the current next frame */
2362           bi0 = from[0];
2363           to_next[0] = bi0;
2364           from += 1;
2365           to_next += 1;
2366           n_left_from -= 1;
2367           n_left_to_next -= 1;
2368
2369           b0 = vlib_get_buffer (vm, bi0);
2370           ip0 = vlib_buffer_get_current (b0);
2371           udp0 = ip4_next_header (ip0);
2372           tcp0 = (tcp_header_t *) udp0;
2373
2374           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2375
2376           vnet_get_config_data (&cm->config_main, &b0->current_config_index,
2377                                 &next0, 0);
2378
2379           if (snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0))
2380             next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2381
2382           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2383
2384           /* verify speculative enqueue, maybe switch current next frame */
2385           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2386                                            to_next, n_left_to_next,
2387                                            bi0, next0);
2388          }
2389
2390       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2391     }
2392
2393   vlib_node_increment_counter (vm, nat44_hairpinning_node.index,
2394                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2395                                pkts_processed);
2396   return frame->n_vectors;
2397 }
2398
2399 VLIB_REGISTER_NODE (nat44_hairpinning_node) = {
2400   .function = nat44_hairpinning_fn,
2401   .name = "nat44-hairpinning",
2402   .vector_size = sizeof (u32),
2403   .type = VLIB_NODE_TYPE_INTERNAL,
2404   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2405   .error_strings = snat_in2out_error_strings,
2406   .n_next_nodes = 2,
2407   .next_nodes = {
2408     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2409     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2410   },
2411 };
2412
2413 VLIB_NODE_FUNCTION_MULTIARCH (nat44_hairpinning_node,
2414                               nat44_hairpinning_fn);
2415
2416 static inline void
2417 nat44_reass_hairpinning (snat_main_t *sm,
2418                          vlib_buffer_t * b0,
2419                          ip4_header_t * ip0,
2420                          u16 sport,
2421                          u16 dport,
2422                          u32 proto0)
2423 {
2424   snat_session_key_t key0, sm0;
2425   snat_session_t * s0;
2426   clib_bihash_kv_8_8_t kv0, value0;
2427   ip_csum_t sum0;
2428   u32 new_dst_addr0 = 0, old_dst_addr0, ti = 0, si;
2429   u16 new_dst_port0, old_dst_port0;
2430   udp_header_t * udp0;
2431   tcp_header_t * tcp0;
2432
2433   key0.addr = ip0->dst_address;
2434   key0.port = dport;
2435   key0.protocol = proto0;
2436   key0.fib_index = sm->outside_fib_index;
2437   kv0.key = key0.as_u64;
2438
2439   udp0 = ip4_next_header (ip0);
2440
2441   /* Check if destination is static mappings */
2442   if (!snat_static_mapping_match(sm, key0, &sm0, 1, 0, 0, 0))
2443     {
2444       new_dst_addr0 = sm0.addr.as_u32;
2445       new_dst_port0 = sm0.port;
2446       vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
2447     }
2448   /* or active sessions */
2449   else
2450     {
2451       if (sm->num_workers > 1)
2452         ti = (clib_net_to_host_u16 (udp0->dst_port) - 1024) / sm->port_per_thread;
2453       else
2454         ti = sm->num_workers;
2455
2456       if (!clib_bihash_search_8_8 (&sm->per_thread_data[ti].out2in, &kv0, &value0))
2457         {
2458           si = value0.value;
2459           s0 = pool_elt_at_index (sm->per_thread_data[ti].sessions, si);
2460           new_dst_addr0 = s0->in2out.addr.as_u32;
2461           new_dst_port0 = s0->in2out.port;
2462           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->in2out.fib_index;
2463         }
2464     }
2465
2466   /* Destination is behind the same NAT, use internal address and port */
2467   if (new_dst_addr0)
2468     {
2469       old_dst_addr0 = ip0->dst_address.as_u32;
2470       ip0->dst_address.as_u32 = new_dst_addr0;
2471       sum0 = ip0->checksum;
2472       sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2473                              ip4_header_t, dst_address);
2474       ip0->checksum = ip_csum_fold (sum0);
2475
2476       old_dst_port0 = dport;
2477       if (PREDICT_TRUE(new_dst_port0 != old_dst_port0 &&
2478                        ip4_is_first_fragment (ip0)))
2479         {
2480           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2481             {
2482               tcp0 = ip4_next_header (ip0);
2483               tcp0->dst = new_dst_port0;
2484               sum0 = tcp0->checksum;
2485               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2486                                      ip4_header_t, dst_address);
2487               sum0 = ip_csum_update (sum0, old_dst_port0, new_dst_port0,
2488                                      ip4_header_t /* cheat */, length);
2489               tcp0->checksum = ip_csum_fold(sum0);
2490             }
2491           else
2492             {
2493               udp0->dst_port = new_dst_port0;
2494               udp0->checksum = 0;
2495             }
2496         }
2497       else
2498         {
2499           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2500             {
2501               tcp0 = ip4_next_header (ip0);
2502               sum0 = tcp0->checksum;
2503               sum0 = ip_csum_update (sum0, old_dst_addr0, new_dst_addr0,
2504                                      ip4_header_t, dst_address);
2505               tcp0->checksum = ip_csum_fold(sum0);
2506             }
2507         }
2508     }
2509 }
2510
2511 static uword
2512 nat44_in2out_reass_node_fn (vlib_main_t * vm,
2513                             vlib_node_runtime_t * node,
2514                             vlib_frame_t * frame)
2515 {
2516   u32 n_left_from, *from, *to_next;
2517   snat_in2out_next_t next_index;
2518   u32 pkts_processed = 0;
2519   snat_main_t *sm = &snat_main;
2520   f64 now = vlib_time_now (vm);
2521   u32 thread_index = vlib_get_thread_index ();
2522   snat_main_per_thread_data_t *per_thread_data =
2523     &sm->per_thread_data[thread_index];
2524   u32 *fragments_to_drop = 0;
2525   u32 *fragments_to_loopback = 0;
2526
2527   from = vlib_frame_vector_args (frame);
2528   n_left_from = frame->n_vectors;
2529   next_index = node->cached_next_index;
2530
2531   while (n_left_from > 0)
2532     {
2533       u32 n_left_to_next;
2534
2535       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2536
2537       while (n_left_from > 0 && n_left_to_next > 0)
2538        {
2539           u32 bi0, sw_if_index0, proto0, rx_fib_index0, new_addr0, old_addr0;
2540           vlib_buffer_t *b0;
2541           u32 next0;
2542           u8 cached0 = 0;
2543           ip4_header_t *ip0;
2544           nat_reass_ip4_t *reass0;
2545           udp_header_t * udp0;
2546           tcp_header_t * tcp0;
2547           snat_session_key_t key0;
2548           clib_bihash_kv_8_8_t kv0, value0;
2549           snat_session_t * s0 = 0;
2550           u16 old_port0, new_port0;
2551           ip_csum_t sum0;
2552
2553           /* speculatively enqueue b0 to the current next frame */
2554           bi0 = from[0];
2555           to_next[0] = bi0;
2556           from += 1;
2557           to_next += 1;
2558           n_left_from -= 1;
2559           n_left_to_next -= 1;
2560
2561           b0 = vlib_get_buffer (vm, bi0);
2562           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2563
2564           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2565           rx_fib_index0 = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2566                                                                sw_if_index0);
2567
2568           if (PREDICT_FALSE (nat_reass_is_drop_frag(0)))
2569             {
2570               next0 = SNAT_IN2OUT_NEXT_DROP;
2571               b0->error = node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT];
2572               goto trace0;
2573             }
2574
2575           ip0 = (ip4_header_t *) vlib_buffer_get_current (b0);
2576           udp0 = ip4_next_header (ip0);
2577           tcp0 = (tcp_header_t *) udp0;
2578           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2579
2580           reass0 = nat_ip4_reass_find_or_create (ip0->src_address,
2581                                                  ip0->dst_address,
2582                                                  ip0->fragment_id,
2583                                                  ip0->protocol,
2584                                                  1,
2585                                                  &fragments_to_drop);
2586
2587           if (PREDICT_FALSE (!reass0))
2588             {
2589               next0 = SNAT_IN2OUT_NEXT_DROP;
2590               b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_REASS];
2591               goto trace0;
2592             }
2593
2594           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2595             {
2596               key0.addr = ip0->src_address;
2597               key0.port = udp0->src_port;
2598               key0.protocol = proto0;
2599               key0.fib_index = rx_fib_index0;
2600               kv0.key = key0.as_u64;
2601
2602               if (clib_bihash_search_8_8 (&per_thread_data->in2out, &kv0, &value0))
2603                 {
2604                   if (PREDICT_FALSE(snat_not_translate(sm, node, sw_if_index0,
2605                       ip0, proto0, rx_fib_index0, thread_index)))
2606                     goto trace0;
2607
2608                   next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0,
2609                                      &s0, node, next0, thread_index);
2610
2611                   if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP))
2612                     goto trace0;
2613
2614                   reass0->sess_index = s0 - per_thread_data->sessions;
2615                 }
2616               else
2617                 {
2618                   s0 = pool_elt_at_index (per_thread_data->sessions,
2619                                           value0.value);
2620                   reass0->sess_index = value0.value;
2621                 }
2622               nat_ip4_reass_get_frags (reass0, &fragments_to_loopback);
2623             }
2624           else
2625             {
2626               if (PREDICT_FALSE (reass0->sess_index == (u32) ~0))
2627                 {
2628                   if (nat_ip4_reass_add_fragment (reass0, bi0))
2629                     {
2630                       b0->error = node->errors[SNAT_IN2OUT_ERROR_MAX_FRAG];
2631                       next0 = SNAT_IN2OUT_NEXT_DROP;
2632                       goto trace0;
2633                     }
2634                   cached0 = 1;
2635                   goto trace0;
2636                 }
2637               s0 = pool_elt_at_index (per_thread_data->sessions,
2638                                       reass0->sess_index);
2639             }
2640
2641           old_addr0 = ip0->src_address.as_u32;
2642           ip0->src_address = s0->out2in.addr;
2643           new_addr0 = ip0->src_address.as_u32;
2644           vnet_buffer(b0)->sw_if_index[VLIB_TX] = s0->out2in.fib_index;
2645
2646           sum0 = ip0->checksum;
2647           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2648                                  ip4_header_t,
2649                                  src_address /* changed member */);
2650           ip0->checksum = ip_csum_fold (sum0);
2651
2652           if (PREDICT_FALSE (ip4_is_first_fragment (ip0)))
2653             {
2654               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2655                 {
2656                   old_port0 = tcp0->src_port;
2657                   tcp0->src_port = s0->out2in.port;
2658                   new_port0 = tcp0->src_port;
2659
2660                   sum0 = tcp0->checksum;
2661                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
2662                                          ip4_header_t,
2663                                          dst_address /* changed member */);
2664                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
2665                                          ip4_header_t /* cheat */,
2666                                          length /* changed member */);
2667                   tcp0->checksum = ip_csum_fold(sum0);
2668                 }
2669               else
2670                 {
2671                   old_port0 = udp0->src_port;
2672                   udp0->src_port = s0->out2in.port;
2673                   udp0->checksum = 0;
2674                 }
2675             }
2676
2677           /* Hairpinning */
2678           nat44_reass_hairpinning (sm, b0, ip0, s0->out2in.port,
2679                                    s0->ext_host_port, proto0);
2680
2681           /* Accounting */
2682           s0->last_heard = now;
2683           s0->total_pkts++;
2684           s0->total_bytes += vlib_buffer_length_in_chain (vm, b0);
2685           /* Per-user LRU list maintenance */
2686           clib_dlist_remove (sm->per_thread_data[thread_index].list_pool,
2687                              s0->per_user_index);
2688           clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool,
2689                               s0->per_user_list_head_index,
2690                               s0->per_user_index);
2691
2692         trace0:
2693           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2694                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2695             {
2696               nat44_in2out_reass_trace_t *t =
2697                  vlib_add_trace (vm, node, b0, sizeof (*t));
2698               t->cached = cached0;
2699               t->sw_if_index = sw_if_index0;
2700               t->next_index = next0;
2701             }
2702
2703           if (cached0)
2704             {
2705               n_left_to_next++;
2706               to_next--;
2707             }
2708           else
2709             {
2710               pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
2711
2712               /* verify speculative enqueue, maybe switch current next frame */
2713               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2714                                                to_next, n_left_to_next,
2715                                                bi0, next0);
2716             }
2717
2718           if (n_left_from == 0 && vec_len (fragments_to_loopback))
2719             {
2720               from = vlib_frame_vector_args (frame);
2721               u32 len = vec_len (fragments_to_loopback);
2722               if (len <= VLIB_FRAME_SIZE)
2723                 {
2724                   clib_memcpy (from, fragments_to_loopback, sizeof (u32) * len);
2725                   n_left_from = len;
2726                   vec_reset_length (fragments_to_loopback);
2727                 }
2728               else
2729                 {
2730                   clib_memcpy (from,
2731                                fragments_to_loopback + (len - VLIB_FRAME_SIZE),
2732                                sizeof (u32) * VLIB_FRAME_SIZE);
2733                   n_left_from = VLIB_FRAME_SIZE;
2734                   _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE;
2735                 }
2736             }
2737        }
2738
2739       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2740     }
2741
2742   vlib_node_increment_counter (vm, nat44_in2out_reass_node.index,
2743                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
2744                                pkts_processed);
2745
2746   nat_send_all_to_node (vm, fragments_to_drop, node,
2747                         &node->errors[SNAT_IN2OUT_ERROR_DROP_FRAGMENT],
2748                         SNAT_IN2OUT_NEXT_DROP);
2749
2750   vec_free (fragments_to_drop);
2751   vec_free (fragments_to_loopback);
2752   return frame->n_vectors;
2753 }
2754
2755 VLIB_REGISTER_NODE (nat44_in2out_reass_node) = {
2756   .function = nat44_in2out_reass_node_fn,
2757   .name = "nat44-in2out-reass",
2758   .vector_size = sizeof (u32),
2759   .format_trace = format_nat44_in2out_reass_trace,
2760   .type = VLIB_NODE_TYPE_INTERNAL,
2761
2762   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
2763   .error_strings = snat_in2out_error_strings,
2764
2765   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
2766   .next_nodes = {
2767     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
2768     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
2769     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
2770     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2771     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
2772   },
2773 };
2774
2775 VLIB_NODE_FUNCTION_MULTIARCH (nat44_in2out_reass_node,
2776                               nat44_in2out_reass_node_fn);
2777
2778 /**************************/
2779 /*** deterministic mode ***/
2780 /**************************/
2781 static uword
2782 snat_det_in2out_node_fn (vlib_main_t * vm,
2783                          vlib_node_runtime_t * node,
2784                          vlib_frame_t * frame)
2785 {
2786   u32 n_left_from, * from, * to_next;
2787   snat_in2out_next_t next_index;
2788   u32 pkts_processed = 0;
2789   snat_main_t * sm = &snat_main;
2790   u32 now = (u32) vlib_time_now (vm);
2791   u32 thread_index = vlib_get_thread_index ();
2792
2793   from = vlib_frame_vector_args (frame);
2794   n_left_from = frame->n_vectors;
2795   next_index = node->cached_next_index;
2796
2797   while (n_left_from > 0)
2798     {
2799       u32 n_left_to_next;
2800
2801       vlib_get_next_frame (vm, node, next_index,
2802                            to_next, n_left_to_next);
2803
2804       while (n_left_from >= 4 && n_left_to_next >= 2)
2805         {
2806           u32 bi0, bi1;
2807           vlib_buffer_t * b0, * b1;
2808           u32 next0, next1;
2809           u32 sw_if_index0, sw_if_index1;
2810           ip4_header_t * ip0, * ip1;
2811           ip_csum_t sum0, sum1;
2812           ip4_address_t new_addr0, old_addr0, new_addr1, old_addr1;
2813           u16 old_port0, new_port0, lo_port0, i0;
2814           u16 old_port1, new_port1, lo_port1, i1;
2815           udp_header_t * udp0, * udp1;
2816           tcp_header_t * tcp0, * tcp1;
2817           u32 proto0, proto1;
2818           snat_det_out_key_t key0, key1;
2819           snat_det_map_t * dm0, * dm1;
2820           snat_det_session_t * ses0 = 0, * ses1 = 0;
2821           u32 rx_fib_index0, rx_fib_index1;
2822           icmp46_header_t * icmp0, * icmp1;
2823
2824           /* Prefetch next iteration. */
2825           {
2826             vlib_buffer_t * p2, * p3;
2827
2828             p2 = vlib_get_buffer (vm, from[2]);
2829             p3 = vlib_get_buffer (vm, from[3]);
2830
2831             vlib_prefetch_buffer_header (p2, LOAD);
2832             vlib_prefetch_buffer_header (p3, LOAD);
2833
2834             CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
2835             CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
2836           }
2837
2838           /* speculatively enqueue b0 and b1 to the current next frame */
2839           to_next[0] = bi0 = from[0];
2840           to_next[1] = bi1 = from[1];
2841           from += 2;
2842           to_next += 2;
2843           n_left_from -= 2;
2844           n_left_to_next -= 2;
2845
2846           b0 = vlib_get_buffer (vm, bi0);
2847           b1 = vlib_get_buffer (vm, bi1);
2848
2849           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
2850           next1 = SNAT_IN2OUT_NEXT_LOOKUP;
2851
2852           ip0 = vlib_buffer_get_current (b0);
2853           udp0 = ip4_next_header (ip0);
2854           tcp0 = (tcp_header_t *) udp0;
2855
2856           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
2857
2858           if (PREDICT_FALSE(ip0->ttl == 1))
2859             {
2860               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2861               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
2862                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
2863                                            0);
2864               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2865               goto trace0;
2866             }
2867
2868           proto0 = ip_proto_to_snat_proto (ip0->protocol);
2869
2870           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
2871             {
2872               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
2873               icmp0 = (icmp46_header_t *) udp0;
2874
2875               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
2876                                   rx_fib_index0, node, next0, thread_index,
2877                                   &ses0, &dm0);
2878               goto trace0;
2879             }
2880
2881           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
2882           if (PREDICT_FALSE(!dm0))
2883             {
2884               clib_warning("no match for internal host %U",
2885                            format_ip4_address, &ip0->src_address);
2886               next0 = SNAT_IN2OUT_NEXT_DROP;
2887               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
2888               goto trace0;
2889             }
2890
2891           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
2892
2893           key0.ext_host_addr = ip0->dst_address;
2894           key0.ext_host_port = tcp0->dst;
2895
2896           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
2897           if (PREDICT_FALSE(!ses0))
2898             {
2899               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
2900                 {
2901                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
2902                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
2903
2904                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
2905                     continue;
2906
2907                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
2908                   break;
2909                 }
2910               if (PREDICT_FALSE(!ses0))
2911                 {
2912                   /* too many sessions for user, send ICMP error packet */
2913
2914                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2915                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
2916                                                ICMP4_destination_unreachable_destination_unreachable_host,
2917                                                0);
2918                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
2919                   goto trace0;
2920                 }
2921             }
2922
2923           new_port0 = ses0->out.out_port;
2924
2925           old_addr0.as_u32 = ip0->src_address.as_u32;
2926           ip0->src_address.as_u32 = new_addr0.as_u32;
2927           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
2928
2929           sum0 = ip0->checksum;
2930           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2931                                  ip4_header_t,
2932                                  src_address /* changed member */);
2933           ip0->checksum = ip_csum_fold (sum0);
2934
2935           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
2936             {
2937               if (tcp0->flags & TCP_FLAG_SYN)
2938                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
2939               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
2940                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2941               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
2942                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
2943               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
2944                 snat_det_ses_close(dm0, ses0);
2945               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
2946                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
2947               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
2948                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
2949
2950               old_port0 = tcp0->src;
2951               tcp0->src = new_port0;
2952
2953               sum0 = tcp0->checksum;
2954               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
2955                                      ip4_header_t,
2956                                      dst_address /* changed member */);
2957               sum0 = ip_csum_update (sum0, old_port0, new_port0,
2958                                      ip4_header_t /* cheat */,
2959                                      length /* changed member */);
2960               tcp0->checksum = ip_csum_fold(sum0);
2961             }
2962           else
2963             {
2964               ses0->state = SNAT_SESSION_UDP_ACTIVE;
2965               old_port0 = udp0->src_port;
2966               udp0->src_port = new_port0;
2967               udp0->checksum = 0;
2968             }
2969
2970           switch(ses0->state)
2971             {
2972             case SNAT_SESSION_UDP_ACTIVE:
2973                 ses0->expire = now + sm->udp_timeout;
2974                 break;
2975             case SNAT_SESSION_TCP_SYN_SENT:
2976             case SNAT_SESSION_TCP_FIN_WAIT:
2977             case SNAT_SESSION_TCP_CLOSE_WAIT:
2978             case SNAT_SESSION_TCP_LAST_ACK:
2979                 ses0->expire = now + sm->tcp_transitory_timeout;
2980                 break;
2981             case SNAT_SESSION_TCP_ESTABLISHED:
2982                 ses0->expire = now + sm->tcp_established_timeout;
2983                 break;
2984             }
2985
2986         trace0:
2987           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
2988                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
2989             {
2990               snat_in2out_trace_t *t =
2991                  vlib_add_trace (vm, node, b0, sizeof (*t));
2992               t->is_slow_path = 0;
2993               t->sw_if_index = sw_if_index0;
2994               t->next_index = next0;
2995               t->session_index = ~0;
2996               if (ses0)
2997                 t->session_index = ses0 - dm0->sessions;
2998             }
2999
3000           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3001
3002           ip1 = vlib_buffer_get_current (b1);
3003           udp1 = ip4_next_header (ip1);
3004           tcp1 = (tcp_header_t *) udp1;
3005
3006           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
3007
3008           if (PREDICT_FALSE(ip1->ttl == 1))
3009             {
3010               vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3011               icmp4_error_set_vnet_buffer (b1, ICMP4_time_exceeded,
3012                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3013                                            0);
3014               next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3015               goto trace1;
3016             }
3017
3018           proto1 = ip_proto_to_snat_proto (ip1->protocol);
3019
3020           if (PREDICT_FALSE(proto1 == SNAT_PROTOCOL_ICMP))
3021             {
3022               rx_fib_index1 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index1);
3023               icmp1 = (icmp46_header_t *) udp1;
3024
3025               next1 = icmp_in2out(sm, b1, ip1, icmp1, sw_if_index1,
3026                                   rx_fib_index1, node, next1, thread_index,
3027                                   &ses1, &dm1);
3028               goto trace1;
3029             }
3030
3031           dm1 = snat_det_map_by_user(sm, &ip1->src_address);
3032           if (PREDICT_FALSE(!dm1))
3033             {
3034               clib_warning("no match for internal host %U",
3035                            format_ip4_address, &ip0->src_address);
3036               next1 = SNAT_IN2OUT_NEXT_DROP;
3037               b1->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3038               goto trace1;
3039             }
3040
3041           snat_det_forward(dm1, &ip1->src_address, &new_addr1, &lo_port1);
3042
3043           key1.ext_host_addr = ip1->dst_address;
3044           key1.ext_host_port = tcp1->dst;
3045
3046           ses1 = snat_det_find_ses_by_in(dm1, &ip1->src_address, tcp1->src, key1);
3047           if (PREDICT_FALSE(!ses1))
3048             {
3049               for (i1 = 0; i1 < dm1->ports_per_host; i1++)
3050                 {
3051                   key1.out_port = clib_host_to_net_u16 (lo_port1 +
3052                     ((i1 + clib_net_to_host_u16 (tcp1->src)) % dm1->ports_per_host));
3053
3054                   if (snat_det_get_ses_by_out (dm1, &ip1->src_address, key1.as_u64))
3055                     continue;
3056
3057                   ses1 = snat_det_ses_create(dm1, &ip1->src_address, tcp1->src, &key1);
3058                   break;
3059                 }
3060               if (PREDICT_FALSE(!ses1))
3061                 {
3062                   /* too many sessions for user, send ICMP error packet */
3063
3064                   vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3065                   icmp4_error_set_vnet_buffer (b1, ICMP4_destination_unreachable,
3066                                                ICMP4_destination_unreachable_destination_unreachable_host,
3067                                                0);
3068                   next1 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3069                   goto trace1;
3070                 }
3071             }
3072
3073           new_port1 = ses1->out.out_port;
3074
3075           old_addr1.as_u32 = ip1->src_address.as_u32;
3076           ip1->src_address.as_u32 = new_addr1.as_u32;
3077           vnet_buffer(b1)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3078
3079           sum1 = ip1->checksum;
3080           sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3081                                  ip4_header_t,
3082                                  src_address /* changed member */);
3083           ip1->checksum = ip_csum_fold (sum1);
3084
3085           if (PREDICT_TRUE(proto1 == SNAT_PROTOCOL_TCP))
3086             {
3087               if (tcp1->flags & TCP_FLAG_SYN)
3088                 ses1->state = SNAT_SESSION_TCP_SYN_SENT;
3089               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_SYN_SENT)
3090                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3091               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_ESTABLISHED)
3092                 ses1->state = SNAT_SESSION_TCP_FIN_WAIT;
3093               else if (tcp1->flags & TCP_FLAG_ACK && ses1->state == SNAT_SESSION_TCP_FIN_WAIT)
3094                 snat_det_ses_close(dm1, ses1);
3095               else if (tcp1->flags & TCP_FLAG_FIN && ses1->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3096                 ses1->state = SNAT_SESSION_TCP_LAST_ACK;
3097               else if (tcp1->flags == 0 && ses1->state == SNAT_SESSION_UNKNOWN)
3098                 ses1->state = SNAT_SESSION_TCP_ESTABLISHED;
3099
3100               old_port1 = tcp1->src;
3101               tcp1->src = new_port1;
3102
3103               sum1 = tcp1->checksum;
3104               sum1 = ip_csum_update (sum1, old_addr1.as_u32, new_addr1.as_u32,
3105                                      ip4_header_t,
3106                                      dst_address /* changed member */);
3107               sum1 = ip_csum_update (sum1, old_port1, new_port1,
3108                                      ip4_header_t /* cheat */,
3109                                      length /* changed member */);
3110               tcp1->checksum = ip_csum_fold(sum1);
3111             }
3112           else
3113             {
3114               ses1->state = SNAT_SESSION_UDP_ACTIVE;
3115               old_port1 = udp1->src_port;
3116               udp1->src_port = new_port1;
3117               udp1->checksum = 0;
3118             }
3119
3120           switch(ses1->state)
3121             {
3122             case SNAT_SESSION_UDP_ACTIVE:
3123                 ses1->expire = now + sm->udp_timeout;
3124                 break;
3125             case SNAT_SESSION_TCP_SYN_SENT:
3126             case SNAT_SESSION_TCP_FIN_WAIT:
3127             case SNAT_SESSION_TCP_CLOSE_WAIT:
3128             case SNAT_SESSION_TCP_LAST_ACK:
3129                 ses1->expire = now + sm->tcp_transitory_timeout;
3130                 break;
3131             case SNAT_SESSION_TCP_ESTABLISHED:
3132                 ses1->expire = now + sm->tcp_established_timeout;
3133                 break;
3134             }
3135
3136         trace1:
3137           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3138                             && (b1->flags & VLIB_BUFFER_IS_TRACED)))
3139             {
3140               snat_in2out_trace_t *t =
3141                  vlib_add_trace (vm, node, b1, sizeof (*t));
3142               t->is_slow_path = 0;
3143               t->sw_if_index = sw_if_index1;
3144               t->next_index = next1;
3145               t->session_index = ~0;
3146               if (ses1)
3147                 t->session_index = ses1 - dm1->sessions;
3148             }
3149
3150           pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP;
3151
3152           /* verify speculative enqueues, maybe switch current next frame */
3153           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
3154                                            to_next, n_left_to_next,
3155                                            bi0, bi1, next0, next1);
3156          }
3157
3158       while (n_left_from > 0 && n_left_to_next > 0)
3159         {
3160           u32 bi0;
3161           vlib_buffer_t * b0;
3162           u32 next0;
3163           u32 sw_if_index0;
3164           ip4_header_t * ip0;
3165           ip_csum_t sum0;
3166           ip4_address_t new_addr0, old_addr0;
3167           u16 old_port0, new_port0, lo_port0, i0;
3168           udp_header_t * udp0;
3169           tcp_header_t * tcp0;
3170           u32 proto0;
3171           snat_det_out_key_t key0;
3172           snat_det_map_t * dm0;
3173           snat_det_session_t * ses0 = 0;
3174           u32 rx_fib_index0;
3175           icmp46_header_t * icmp0;
3176
3177           /* speculatively enqueue b0 to the current next frame */
3178           bi0 = from[0];
3179           to_next[0] = bi0;
3180           from += 1;
3181           to_next += 1;
3182           n_left_from -= 1;
3183           n_left_to_next -= 1;
3184
3185           b0 = vlib_get_buffer (vm, bi0);
3186           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3187
3188           ip0 = vlib_buffer_get_current (b0);
3189           udp0 = ip4_next_header (ip0);
3190           tcp0 = (tcp_header_t *) udp0;
3191
3192           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3193
3194           if (PREDICT_FALSE(ip0->ttl == 1))
3195             {
3196               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3197               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
3198                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
3199                                            0);
3200               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3201               goto trace00;
3202             }
3203
3204           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3205
3206           if (PREDICT_FALSE(proto0 == SNAT_PROTOCOL_ICMP))
3207             {
3208               rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3209               icmp0 = (icmp46_header_t *) udp0;
3210
3211               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
3212                                   rx_fib_index0, node, next0, thread_index,
3213                                   &ses0, &dm0);
3214               goto trace00;
3215             }
3216
3217           dm0 = snat_det_map_by_user(sm, &ip0->src_address);
3218           if (PREDICT_FALSE(!dm0))
3219             {
3220               clib_warning("no match for internal host %U",
3221                            format_ip4_address, &ip0->src_address);
3222               next0 = SNAT_IN2OUT_NEXT_DROP;
3223               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3224               goto trace00;
3225             }
3226
3227           snat_det_forward(dm0, &ip0->src_address, &new_addr0, &lo_port0);
3228
3229           key0.ext_host_addr = ip0->dst_address;
3230           key0.ext_host_port = tcp0->dst;
3231
3232           ses0 = snat_det_find_ses_by_in(dm0, &ip0->src_address, tcp0->src, key0);
3233           if (PREDICT_FALSE(!ses0))
3234             {
3235               for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3236                 {
3237                   key0.out_port = clib_host_to_net_u16 (lo_port0 +
3238                     ((i0 + clib_net_to_host_u16 (tcp0->src)) % dm0->ports_per_host));
3239
3240                   if (snat_det_get_ses_by_out (dm0, &ip0->src_address, key0.as_u64))
3241                     continue;
3242
3243                   ses0 = snat_det_ses_create(dm0, &ip0->src_address, tcp0->src, &key0);
3244                   break;
3245                 }
3246               if (PREDICT_FALSE(!ses0))
3247                 {
3248                   /* too many sessions for user, send ICMP error packet */
3249
3250                   vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
3251                   icmp4_error_set_vnet_buffer (b0, ICMP4_destination_unreachable,
3252                                                ICMP4_destination_unreachable_destination_unreachable_host,
3253                                                0);
3254                   next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
3255                   goto trace00;
3256                 }
3257             }
3258
3259           new_port0 = ses0->out.out_port;
3260
3261           old_addr0.as_u32 = ip0->src_address.as_u32;
3262           ip0->src_address.as_u32 = new_addr0.as_u32;
3263           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
3264
3265           sum0 = ip0->checksum;
3266           sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3267                                  ip4_header_t,
3268                                  src_address /* changed member */);
3269           ip0->checksum = ip_csum_fold (sum0);
3270
3271           if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
3272             {
3273               if (tcp0->flags & TCP_FLAG_SYN)
3274                 ses0->state = SNAT_SESSION_TCP_SYN_SENT;
3275               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_SYN_SENT)
3276                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3277               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_ESTABLISHED)
3278                 ses0->state = SNAT_SESSION_TCP_FIN_WAIT;
3279               else if (tcp0->flags & TCP_FLAG_ACK && ses0->state == SNAT_SESSION_TCP_FIN_WAIT)
3280                 snat_det_ses_close(dm0, ses0);
3281               else if (tcp0->flags & TCP_FLAG_FIN && ses0->state == SNAT_SESSION_TCP_CLOSE_WAIT)
3282                 ses0->state = SNAT_SESSION_TCP_LAST_ACK;
3283               else if (tcp0->flags == 0 && ses0->state == SNAT_SESSION_UNKNOWN)
3284                 ses0->state = SNAT_SESSION_TCP_ESTABLISHED;
3285
3286               old_port0 = tcp0->src;
3287               tcp0->src = new_port0;
3288
3289               sum0 = tcp0->checksum;
3290               sum0 = ip_csum_update (sum0, old_addr0.as_u32, new_addr0.as_u32,
3291                                      ip4_header_t,
3292                                      dst_address /* changed member */);
3293               sum0 = ip_csum_update (sum0, old_port0, new_port0,
3294                                      ip4_header_t /* cheat */,
3295                                      length /* changed member */);
3296               tcp0->checksum = ip_csum_fold(sum0);
3297             }
3298           else
3299             {
3300               ses0->state = SNAT_SESSION_UDP_ACTIVE;
3301               old_port0 = udp0->src_port;
3302               udp0->src_port = new_port0;
3303               udp0->checksum = 0;
3304             }
3305
3306           switch(ses0->state)
3307             {
3308             case SNAT_SESSION_UDP_ACTIVE:
3309                 ses0->expire = now + sm->udp_timeout;
3310                 break;
3311             case SNAT_SESSION_TCP_SYN_SENT:
3312             case SNAT_SESSION_TCP_FIN_WAIT:
3313             case SNAT_SESSION_TCP_CLOSE_WAIT:
3314             case SNAT_SESSION_TCP_LAST_ACK:
3315                 ses0->expire = now + sm->tcp_transitory_timeout;
3316                 break;
3317             case SNAT_SESSION_TCP_ESTABLISHED:
3318                 ses0->expire = now + sm->tcp_established_timeout;
3319                 break;
3320             }
3321
3322         trace00:
3323           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
3324                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3325             {
3326               snat_in2out_trace_t *t =
3327                  vlib_add_trace (vm, node, b0, sizeof (*t));
3328               t->is_slow_path = 0;
3329               t->sw_if_index = sw_if_index0;
3330               t->next_index = next0;
3331               t->session_index = ~0;
3332               if (ses0)
3333                 t->session_index = ses0 - dm0->sessions;
3334             }
3335
3336           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3337
3338           /* verify speculative enqueue, maybe switch current next frame */
3339           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3340                                            to_next, n_left_to_next,
3341                                            bi0, next0);
3342         }
3343
3344       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3345     }
3346
3347   vlib_node_increment_counter (vm, snat_det_in2out_node.index,
3348                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3349                                pkts_processed);
3350   return frame->n_vectors;
3351 }
3352
3353 VLIB_REGISTER_NODE (snat_det_in2out_node) = {
3354   .function = snat_det_in2out_node_fn,
3355   .name = "nat44-det-in2out",
3356   .vector_size = sizeof (u32),
3357   .format_trace = format_snat_in2out_trace,
3358   .type = VLIB_NODE_TYPE_INTERNAL,
3359
3360   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3361   .error_strings = snat_in2out_error_strings,
3362
3363   .runtime_data_bytes = sizeof (snat_runtime_t),
3364
3365   .n_next_nodes = 3,
3366
3367   /* edit / add dispositions here */
3368   .next_nodes = {
3369     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3370     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3371     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
3372   },
3373 };
3374
3375 VLIB_NODE_FUNCTION_MULTIARCH (snat_det_in2out_node, snat_det_in2out_node_fn);
3376
3377 /**
3378  * Get address and port values to be used for ICMP packet translation
3379  * and create session if needed
3380  *
3381  * @param[in,out] sm             NAT main
3382  * @param[in,out] node           NAT node runtime
3383  * @param[in] thread_index       thread index
3384  * @param[in,out] b0             buffer containing packet to be translated
3385  * @param[out] p_proto           protocol used for matching
3386  * @param[out] p_value           address and port after NAT translation
3387  * @param[out] p_dont_translate  if packet should not be translated
3388  * @param d                      optional parameter
3389  * @param e                      optional parameter
3390  */
3391 u32 icmp_match_in2out_det(snat_main_t *sm, vlib_node_runtime_t *node,
3392                           u32 thread_index, vlib_buffer_t *b0,
3393                           ip4_header_t *ip0, u8 *p_proto,
3394                           snat_session_key_t *p_value,
3395                           u8 *p_dont_translate, void *d, void *e)
3396 {
3397   icmp46_header_t *icmp0;
3398   u32 sw_if_index0;
3399   u32 rx_fib_index0;
3400   u8 protocol;
3401   snat_det_out_key_t key0;
3402   u8 dont_translate = 0;
3403   u32 next0 = ~0;
3404   icmp_echo_header_t *echo0, *inner_echo0 = 0;
3405   ip4_header_t *inner_ip0;
3406   void *l4_header = 0;
3407   icmp46_header_t *inner_icmp0;
3408   snat_det_map_t * dm0 = 0;
3409   ip4_address_t new_addr0;
3410   u16 lo_port0, i0;
3411   snat_det_session_t * ses0 = 0;
3412   ip4_address_t in_addr;
3413   u16 in_port;
3414
3415   icmp0 = (icmp46_header_t *) ip4_next_header (ip0);
3416   echo0 = (icmp_echo_header_t *)(icmp0+1);
3417   sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3418   rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
3419
3420   if (!icmp_is_error_message (icmp0))
3421     {
3422       protocol = SNAT_PROTOCOL_ICMP;
3423       in_addr = ip0->src_address;
3424       in_port = echo0->identifier;
3425     }
3426   else
3427     {
3428       inner_ip0 = (ip4_header_t *)(echo0+1);
3429       l4_header = ip4_next_header (inner_ip0);
3430       protocol = ip_proto_to_snat_proto (inner_ip0->protocol);
3431       in_addr = inner_ip0->dst_address;
3432       switch (protocol)
3433         {
3434         case SNAT_PROTOCOL_ICMP:
3435           inner_icmp0 = (icmp46_header_t*)l4_header;
3436           inner_echo0 = (icmp_echo_header_t *)(inner_icmp0+1);
3437           in_port = inner_echo0->identifier;
3438           break;
3439         case SNAT_PROTOCOL_UDP:
3440         case SNAT_PROTOCOL_TCP:
3441           in_port = ((tcp_udp_header_t*)l4_header)->dst_port;
3442           break;
3443         default:
3444           b0->error = node->errors[SNAT_IN2OUT_ERROR_UNSUPPORTED_PROTOCOL];
3445           next0 = SNAT_IN2OUT_NEXT_DROP;
3446           goto out;
3447         }
3448     }
3449
3450   dm0 = snat_det_map_by_user(sm, &in_addr);
3451   if (PREDICT_FALSE(!dm0))
3452     {
3453       clib_warning("no match for internal host %U",
3454                    format_ip4_address, &in_addr);
3455       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3456           IP_PROTOCOL_ICMP, rx_fib_index0)))
3457         {
3458           dont_translate = 1;
3459           goto out;
3460         }
3461       next0 = SNAT_IN2OUT_NEXT_DROP;
3462       b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
3463       goto out;
3464     }
3465
3466   snat_det_forward(dm0, &in_addr, &new_addr0, &lo_port0);
3467
3468   key0.ext_host_addr = ip0->dst_address;
3469   key0.ext_host_port = 0;
3470
3471   ses0 = snat_det_find_ses_by_in(dm0, &in_addr, in_port, key0);
3472   if (PREDICT_FALSE(!ses0))
3473     {
3474       if (PREDICT_FALSE(snat_not_translate_fast(sm, node, sw_if_index0, ip0,
3475           IP_PROTOCOL_ICMP, rx_fib_index0)))
3476         {
3477           dont_translate = 1;
3478           goto out;
3479         }
3480       if (icmp0->type != ICMP4_echo_request)
3481         {
3482           b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3483           next0 = SNAT_IN2OUT_NEXT_DROP;
3484           goto out;
3485         }
3486       for (i0 = 0; i0 < dm0->ports_per_host; i0++)
3487         {
3488           key0.out_port = clib_host_to_net_u16 (lo_port0 +
3489             ((i0 + clib_net_to_host_u16 (echo0->identifier)) % dm0->ports_per_host));
3490
3491           if (snat_det_get_ses_by_out (dm0, &in_addr, key0.as_u64))
3492             continue;
3493
3494           ses0 = snat_det_ses_create(dm0, &in_addr, echo0->identifier, &key0);
3495           break;
3496         }
3497       if (PREDICT_FALSE(!ses0))
3498         {
3499           next0 = SNAT_IN2OUT_NEXT_DROP;
3500           b0->error = node->errors[SNAT_IN2OUT_ERROR_OUT_OF_PORTS];
3501           goto out;
3502         }
3503     }
3504
3505   if (PREDICT_FALSE(icmp0->type != ICMP4_echo_request &&
3506                     !icmp_is_error_message (icmp0)))
3507     {
3508       b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_ICMP_TYPE];
3509       next0 = SNAT_IN2OUT_NEXT_DROP;
3510       goto out;
3511     }
3512
3513   u32 now = (u32) vlib_time_now (sm->vlib_main);
3514
3515   ses0->state = SNAT_SESSION_ICMP_ACTIVE;
3516   ses0->expire = now + sm->icmp_timeout;
3517
3518 out:
3519   *p_proto = protocol;
3520   if (ses0)
3521     {
3522       p_value->addr = new_addr0;
3523       p_value->fib_index = sm->outside_fib_index;
3524       p_value->port = ses0->out.out_port;
3525     }
3526   *p_dont_translate = dont_translate;
3527   if (d)
3528     *(snat_det_session_t**)d = ses0;
3529   if (e)
3530     *(snat_det_map_t**)e = dm0;
3531   return next0;
3532 }
3533
3534 /**********************/
3535 /*** worker handoff ***/
3536 /**********************/
3537 static inline uword
3538 snat_in2out_worker_handoff_fn_inline (vlib_main_t * vm,
3539                                       vlib_node_runtime_t * node,
3540                                       vlib_frame_t * frame,
3541                                       u8 is_output)
3542 {
3543   snat_main_t *sm = &snat_main;
3544   vlib_thread_main_t *tm = vlib_get_thread_main ();
3545   u32 n_left_from, *from, *to_next = 0;
3546   static __thread vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index;
3547   static __thread vlib_frame_queue_t **congested_handoff_queue_by_worker_index
3548     = 0;
3549   vlib_frame_queue_elt_t *hf = 0;
3550   vlib_frame_t *f = 0;
3551   int i;
3552   u32 n_left_to_next_worker = 0, *to_next_worker = 0;
3553   u32 next_worker_index = 0;
3554   u32 current_worker_index = ~0;
3555   u32 thread_index = vlib_get_thread_index ();
3556   u32 fq_index;
3557   u32 to_node_index;
3558
3559   ASSERT (vec_len (sm->workers));
3560
3561   if (is_output)
3562     {
3563       fq_index = sm->fq_in2out_output_index;
3564       to_node_index = sm->in2out_output_node_index;
3565     }
3566   else
3567     {
3568       fq_index = sm->fq_in2out_index;
3569       to_node_index = sm->in2out_node_index;
3570     }
3571
3572   if (PREDICT_FALSE (handoff_queue_elt_by_worker_index == 0))
3573     {
3574       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
3575
3576       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
3577                                sm->first_worker_index + sm->num_workers - 1,
3578                                (vlib_frame_queue_t *) (~0));
3579     }
3580
3581   from = vlib_frame_vector_args (frame);
3582   n_left_from = frame->n_vectors;
3583
3584   while (n_left_from > 0)
3585     {
3586       u32 bi0;
3587       vlib_buffer_t *b0;
3588       u32 sw_if_index0;
3589       u32 rx_fib_index0;
3590       ip4_header_t * ip0;
3591       u8 do_handoff;
3592
3593       bi0 = from[0];
3594       from += 1;
3595       n_left_from -= 1;
3596
3597       b0 = vlib_get_buffer (vm, bi0);
3598
3599       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3600       rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
3601
3602       ip0 = vlib_buffer_get_current (b0);
3603
3604       next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0);
3605
3606       if (PREDICT_FALSE (next_worker_index != thread_index))
3607         {
3608           do_handoff = 1;
3609
3610           if (next_worker_index != current_worker_index)
3611             {
3612               if (hf)
3613                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3614
3615               hf = vlib_get_worker_handoff_queue_elt (fq_index,
3616                                                       next_worker_index,
3617                                                       handoff_queue_elt_by_worker_index);
3618
3619               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
3620               to_next_worker = &hf->buffer_index[hf->n_vectors];
3621               current_worker_index = next_worker_index;
3622             }
3623
3624           /* enqueue to correct worker thread */
3625           to_next_worker[0] = bi0;
3626           to_next_worker++;
3627           n_left_to_next_worker--;
3628
3629           if (n_left_to_next_worker == 0)
3630             {
3631               hf->n_vectors = VLIB_FRAME_SIZE;
3632               vlib_put_frame_queue_elt (hf);
3633               current_worker_index = ~0;
3634               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
3635               hf = 0;
3636             }
3637         }
3638       else
3639         {
3640           do_handoff = 0;
3641           /* if this is 1st frame */
3642           if (!f)
3643             {
3644               f = vlib_get_frame_to_node (vm, to_node_index);
3645               to_next = vlib_frame_vector_args (f);
3646             }
3647
3648           to_next[0] = bi0;
3649           to_next += 1;
3650           f->n_vectors++;
3651         }
3652
3653       if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
3654                          && (b0->flags & VLIB_BUFFER_IS_TRACED)))
3655         {
3656           snat_in2out_worker_handoff_trace_t *t =
3657             vlib_add_trace (vm, node, b0, sizeof (*t));
3658           t->next_worker_index = next_worker_index;
3659           t->do_handoff = do_handoff;
3660         }
3661     }
3662
3663   if (f)
3664     vlib_put_frame_to_node (vm, to_node_index, f);
3665
3666   if (hf)
3667     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
3668
3669   /* Ship frames to the worker nodes */
3670   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
3671     {
3672       if (handoff_queue_elt_by_worker_index[i])
3673         {
3674           hf = handoff_queue_elt_by_worker_index[i];
3675           /*
3676            * It works better to let the handoff node
3677            * rate-adapt, always ship the handoff queue element.
3678            */
3679           if (1 || hf->n_vectors == hf->last_n_vectors)
3680             {
3681               vlib_put_frame_queue_elt (hf);
3682               handoff_queue_elt_by_worker_index[i] = 0;
3683             }
3684           else
3685             hf->last_n_vectors = hf->n_vectors;
3686         }
3687       congested_handoff_queue_by_worker_index[i] =
3688         (vlib_frame_queue_t *) (~0);
3689     }
3690   hf = 0;
3691   current_worker_index = ~0;
3692   return frame->n_vectors;
3693 }
3694
3695 static uword
3696 snat_in2out_worker_handoff_fn (vlib_main_t * vm,
3697                                vlib_node_runtime_t * node,
3698                                vlib_frame_t * frame)
3699 {
3700   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 0);
3701 }
3702
3703 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
3704   .function = snat_in2out_worker_handoff_fn,
3705   .name = "nat44-in2out-worker-handoff",
3706   .vector_size = sizeof (u32),
3707   .format_trace = format_snat_in2out_worker_handoff_trace,
3708   .type = VLIB_NODE_TYPE_INTERNAL,
3709
3710   .n_next_nodes = 1,
3711
3712   .next_nodes = {
3713     [0] = "error-drop",
3714   },
3715 };
3716
3717 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_worker_handoff_node,
3718                               snat_in2out_worker_handoff_fn);
3719
3720 static uword
3721 snat_in2out_output_worker_handoff_fn (vlib_main_t * vm,
3722                                       vlib_node_runtime_t * node,
3723                                       vlib_frame_t * frame)
3724 {
3725   return snat_in2out_worker_handoff_fn_inline (vm, node, frame, 1);
3726 }
3727
3728 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
3729   .function = snat_in2out_output_worker_handoff_fn,
3730   .name = "nat44-in2out-output-worker-handoff",
3731   .vector_size = sizeof (u32),
3732   .format_trace = format_snat_in2out_worker_handoff_trace,
3733   .type = VLIB_NODE_TYPE_INTERNAL,
3734
3735   .n_next_nodes = 1,
3736
3737   .next_nodes = {
3738     [0] = "error-drop",
3739   },
3740 };
3741
3742 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_output_worker_handoff_node,
3743                               snat_in2out_output_worker_handoff_fn);
3744
3745 static_always_inline int
3746 is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
3747 {
3748   snat_address_t * ap;
3749   clib_bihash_kv_8_8_t kv, value;
3750   snat_session_key_t m_key;
3751
3752   vec_foreach (ap, sm->addresses)
3753     {
3754       if (ap->addr.as_u32 == dst_addr->as_u32)
3755         return 1;
3756     }
3757
3758   m_key.addr.as_u32 = dst_addr->as_u32;
3759   m_key.fib_index = sm->outside_fib_index;
3760   m_key.port = 0;
3761   m_key.protocol = 0;
3762   kv.key = m_key.as_u64;
3763   if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
3764     return 1;
3765
3766   return 0;
3767 }
3768
3769 static uword
3770 snat_hairpin_dst_fn (vlib_main_t * vm,
3771                      vlib_node_runtime_t * node,
3772                      vlib_frame_t * frame)
3773 {
3774   u32 n_left_from, * from, * to_next;
3775   snat_in2out_next_t next_index;
3776   u32 pkts_processed = 0;
3777   snat_main_t * sm = &snat_main;
3778
3779   from = vlib_frame_vector_args (frame);
3780   n_left_from = frame->n_vectors;
3781   next_index = node->cached_next_index;
3782
3783   while (n_left_from > 0)
3784     {
3785       u32 n_left_to_next;
3786
3787       vlib_get_next_frame (vm, node, next_index,
3788                            to_next, n_left_to_next);
3789
3790       while (n_left_from > 0 && n_left_to_next > 0)
3791         {
3792           u32 bi0;
3793           vlib_buffer_t * b0;
3794           u32 next0;
3795           ip4_header_t * ip0;
3796           u32 proto0;
3797
3798           /* speculatively enqueue b0 to the current next frame */
3799           bi0 = from[0];
3800           to_next[0] = bi0;
3801           from += 1;
3802           to_next += 1;
3803           n_left_from -= 1;
3804           n_left_to_next -= 1;
3805
3806           b0 = vlib_get_buffer (vm, bi0);
3807           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
3808           ip0 = vlib_buffer_get_current (b0);
3809
3810           proto0 = ip_proto_to_snat_proto (ip0->protocol);
3811
3812           vnet_buffer (b0)->snat.flags = 0;
3813           if (PREDICT_FALSE (is_hairpinning (sm, &ip0->dst_address)))
3814             {
3815               if (proto0 == SNAT_PROTOCOL_TCP || proto0 == SNAT_PROTOCOL_UDP)
3816                 {
3817                   udp_header_t * udp0 = ip4_next_header (ip0);
3818                   tcp_header_t * tcp0 = (tcp_header_t *) udp0;
3819
3820                   snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
3821                 }
3822               else if (proto0 == SNAT_PROTOCOL_ICMP)
3823                 {
3824                   icmp46_header_t * icmp0 = ip4_next_header (ip0);
3825
3826                   snat_icmp_hairpinning (sm, b0, ip0, icmp0);
3827                 }
3828               else
3829                 {
3830                   snat_hairpinning_unknown_proto (sm, b0, ip0);
3831                 }
3832
3833               vnet_buffer (b0)->snat.flags = SNAT_FLAG_HAIRPINNING;
3834             }
3835
3836           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3837
3838           /* verify speculative enqueue, maybe switch current next frame */
3839           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3840                                            to_next, n_left_to_next,
3841                                            bi0, next0);
3842          }
3843
3844       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3845     }
3846
3847   vlib_node_increment_counter (vm, snat_hairpin_dst_node.index,
3848                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3849                                pkts_processed);
3850   return frame->n_vectors;
3851 }
3852
3853 VLIB_REGISTER_NODE (snat_hairpin_dst_node) = {
3854   .function = snat_hairpin_dst_fn,
3855   .name = "nat44-hairpin-dst",
3856   .vector_size = sizeof (u32),
3857   .type = VLIB_NODE_TYPE_INTERNAL,
3858   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3859   .error_strings = snat_in2out_error_strings,
3860   .n_next_nodes = 2,
3861   .next_nodes = {
3862     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
3863     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
3864   },
3865 };
3866
3867 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_dst_node,
3868                               snat_hairpin_dst_fn);
3869
3870 static uword
3871 snat_hairpin_src_fn (vlib_main_t * vm,
3872                      vlib_node_runtime_t * node,
3873                      vlib_frame_t * frame)
3874 {
3875   u32 n_left_from, * from, * to_next;
3876   snat_in2out_next_t next_index;
3877   u32 pkts_processed = 0;
3878   snat_main_t *sm = &snat_main;
3879
3880   from = vlib_frame_vector_args (frame);
3881   n_left_from = frame->n_vectors;
3882   next_index = node->cached_next_index;
3883
3884   while (n_left_from > 0)
3885     {
3886       u32 n_left_to_next;
3887
3888       vlib_get_next_frame (vm, node, next_index,
3889                            to_next, n_left_to_next);
3890
3891       while (n_left_from > 0 && n_left_to_next > 0)
3892         {
3893           u32 bi0;
3894           vlib_buffer_t * b0;
3895           u32 next0;
3896           snat_interface_t *i;
3897           u32 sw_if_index0;
3898
3899           /* speculatively enqueue b0 to the current next frame */
3900           bi0 = from[0];
3901           to_next[0] = bi0;
3902           from += 1;
3903           to_next += 1;
3904           n_left_from -= 1;
3905           n_left_to_next -= 1;
3906
3907           b0 = vlib_get_buffer (vm, bi0);
3908           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
3909           next0 = SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT;
3910
3911           pool_foreach (i, sm->output_feature_interfaces,
3912           ({
3913             /* Only packets from NAT inside interface */
3914             if ((nat_interface_is_inside(i)) && (sw_if_index0 == i->sw_if_index))
3915               {
3916                 if (PREDICT_FALSE ((vnet_buffer (b0)->snat.flags) &
3917                                     SNAT_FLAG_HAIRPINNING))
3918                   {
3919                     if (PREDICT_TRUE (sm->num_workers > 1))
3920                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH;
3921                     else
3922                       next0 = SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT;
3923                   }
3924                 break;
3925               }
3926           }));
3927
3928           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
3929
3930           /* verify speculative enqueue, maybe switch current next frame */
3931           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
3932                                            to_next, n_left_to_next,
3933                                            bi0, next0);
3934          }
3935
3936       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
3937     }
3938
3939   vlib_node_increment_counter (vm, snat_hairpin_src_node.index,
3940                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
3941                                pkts_processed);
3942   return frame->n_vectors;
3943 }
3944
3945 VLIB_REGISTER_NODE (snat_hairpin_src_node) = {
3946   .function = snat_hairpin_src_fn,
3947   .name = "nat44-hairpin-src",
3948   .vector_size = sizeof (u32),
3949   .type = VLIB_NODE_TYPE_INTERNAL,
3950   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
3951   .error_strings = snat_in2out_error_strings,
3952   .n_next_nodes = SNAT_HAIRPIN_SRC_N_NEXT,
3953   .next_nodes = {
3954      [SNAT_HAIRPIN_SRC_NEXT_DROP] = "error-drop",
3955      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT] = "nat44-in2out-output",
3956      [SNAT_HAIRPIN_SRC_NEXT_INTERFACE_OUTPUT] = "interface-output",
3957      [SNAT_HAIRPIN_SRC_NEXT_SNAT_IN2OUT_WH] = "nat44-in2out-output-worker-handoff",
3958   },
3959 };
3960
3961 VLIB_NODE_FUNCTION_MULTIARCH (snat_hairpin_src_node,
3962                               snat_hairpin_src_fn);
3963
3964 static uword
3965 snat_in2out_fast_static_map_fn (vlib_main_t * vm,
3966                                 vlib_node_runtime_t * node,
3967                                 vlib_frame_t * frame)
3968 {
3969   u32 n_left_from, * from, * to_next;
3970   snat_in2out_next_t next_index;
3971   u32 pkts_processed = 0;
3972   snat_main_t * sm = &snat_main;
3973   u32 stats_node_index;
3974
3975   stats_node_index = snat_in2out_fast_node.index;
3976
3977   from = vlib_frame_vector_args (frame);
3978   n_left_from = frame->n_vectors;
3979   next_index = node->cached_next_index;
3980
3981   while (n_left_from > 0)
3982     {
3983       u32 n_left_to_next;
3984
3985       vlib_get_next_frame (vm, node, next_index,
3986                            to_next, n_left_to_next);
3987
3988       while (n_left_from > 0 && n_left_to_next > 0)
3989         {
3990           u32 bi0;
3991           vlib_buffer_t * b0;
3992           u32 next0;
3993           u32 sw_if_index0;
3994           ip4_header_t * ip0;
3995           ip_csum_t sum0;
3996           u32 new_addr0, old_addr0;
3997           u16 old_port0, new_port0;
3998           udp_header_t * udp0;
3999           tcp_header_t * tcp0;
4000           icmp46_header_t * icmp0;
4001           snat_session_key_t key0, sm0;
4002           u32 proto0;
4003           u32 rx_fib_index0;
4004
4005           /* speculatively enqueue b0 to the current next frame */
4006           bi0 = from[0];
4007           to_next[0] = bi0;
4008           from += 1;
4009           to_next += 1;
4010           n_left_from -= 1;
4011           n_left_to_next -= 1;
4012
4013           b0 = vlib_get_buffer (vm, bi0);
4014           next0 = SNAT_IN2OUT_NEXT_LOOKUP;
4015
4016           ip0 = vlib_buffer_get_current (b0);
4017           udp0 = ip4_next_header (ip0);
4018           tcp0 = (tcp_header_t *) udp0;
4019           icmp0 = (icmp46_header_t *) udp0;
4020
4021           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
4022           rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index(sw_if_index0);
4023
4024           if (PREDICT_FALSE(ip0->ttl == 1))
4025             {
4026               vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
4027               icmp4_error_set_vnet_buffer (b0, ICMP4_time_exceeded,
4028                                            ICMP4_time_exceeded_ttl_exceeded_in_transit,
4029                                            0);
4030               next0 = SNAT_IN2OUT_NEXT_ICMP_ERROR;
4031               goto trace0;
4032             }
4033
4034           proto0 = ip_proto_to_snat_proto (ip0->protocol);
4035
4036           if (PREDICT_FALSE (proto0 == ~0))
4037               goto trace0;
4038
4039           if (PREDICT_FALSE (proto0 == SNAT_PROTOCOL_ICMP))
4040             {
4041               next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0,
4042                                   rx_fib_index0, node, next0, ~0, 0, 0);
4043               goto trace0;
4044             }
4045
4046           key0.addr = ip0->src_address;
4047           key0.protocol = proto0;
4048           key0.port = udp0->src_port;
4049           key0.fib_index = rx_fib_index0;
4050
4051           if (snat_static_mapping_match(sm, key0, &sm0, 0, 0, 0, 0))
4052             {
4053               b0->error = node->errors[SNAT_IN2OUT_ERROR_NO_TRANSLATION];
4054               next0= SNAT_IN2OUT_NEXT_DROP;
4055               goto trace0;
4056             }
4057
4058           new_addr0 = sm0.addr.as_u32;
4059           new_port0 = sm0.port;
4060           vnet_buffer(b0)->sw_if_index[VLIB_TX] = sm0.fib_index;
4061           old_addr0 = ip0->src_address.as_u32;
4062           ip0->src_address.as_u32 = new_addr0;
4063
4064           sum0 = ip0->checksum;
4065           sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4066                                  ip4_header_t,
4067                                  src_address /* changed member */);
4068           ip0->checksum = ip_csum_fold (sum0);
4069
4070           if (PREDICT_FALSE(new_port0 != udp0->dst_port))
4071             {
4072               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4073                 {
4074                   old_port0 = tcp0->src_port;
4075                   tcp0->src_port = new_port0;
4076
4077                   sum0 = tcp0->checksum;
4078                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4079                                          ip4_header_t,
4080                                          dst_address /* changed member */);
4081                   sum0 = ip_csum_update (sum0, old_port0, new_port0,
4082                                          ip4_header_t /* cheat */,
4083                                          length /* changed member */);
4084                   tcp0->checksum = ip_csum_fold(sum0);
4085                 }
4086               else
4087                 {
4088                   old_port0 = udp0->src_port;
4089                   udp0->src_port = new_port0;
4090                   udp0->checksum = 0;
4091                 }
4092             }
4093           else
4094             {
4095               if (PREDICT_TRUE(proto0 == SNAT_PROTOCOL_TCP))
4096                 {
4097                   sum0 = tcp0->checksum;
4098                   sum0 = ip_csum_update (sum0, old_addr0, new_addr0,
4099                                          ip4_header_t,
4100                                          dst_address /* changed member */);
4101                   tcp0->checksum = ip_csum_fold(sum0);
4102                 }
4103             }
4104
4105           /* Hairpinning */
4106           snat_hairpinning (sm, b0, ip0, udp0, tcp0, proto0);
4107
4108         trace0:
4109           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
4110                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
4111             {
4112               snat_in2out_trace_t *t =
4113                  vlib_add_trace (vm, node, b0, sizeof (*t));
4114               t->sw_if_index = sw_if_index0;
4115               t->next_index = next0;
4116             }
4117
4118           pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP;
4119
4120           /* verify speculative enqueue, maybe switch current next frame */
4121           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
4122                                            to_next, n_left_to_next,
4123                                            bi0, next0);
4124         }
4125
4126       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
4127     }
4128
4129   vlib_node_increment_counter (vm, stats_node_index,
4130                                SNAT_IN2OUT_ERROR_IN2OUT_PACKETS,
4131                                pkts_processed);
4132   return frame->n_vectors;
4133 }
4134
4135
4136 VLIB_REGISTER_NODE (snat_in2out_fast_node) = {
4137   .function = snat_in2out_fast_static_map_fn,
4138   .name = "nat44-in2out-fast",
4139   .vector_size = sizeof (u32),
4140   .format_trace = format_snat_in2out_fast_trace,
4141   .type = VLIB_NODE_TYPE_INTERNAL,
4142
4143   .n_errors = ARRAY_LEN(snat_in2out_error_strings),
4144   .error_strings = snat_in2out_error_strings,
4145
4146   .runtime_data_bytes = sizeof (snat_runtime_t),
4147
4148   .n_next_nodes = SNAT_IN2OUT_N_NEXT,
4149
4150   /* edit / add dispositions here */
4151   .next_nodes = {
4152     [SNAT_IN2OUT_NEXT_DROP] = "error-drop",
4153     [SNAT_IN2OUT_NEXT_LOOKUP] = "ip4-lookup",
4154     [SNAT_IN2OUT_NEXT_SLOW_PATH] = "nat44-in2out-slowpath",
4155     [SNAT_IN2OUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
4156     [SNAT_IN2OUT_NEXT_REASS] = "nat44-in2out-reass",
4157   },
4158 };
4159
4160 VLIB_NODE_FUNCTION_MULTIARCH (snat_in2out_fast_node, snat_in2out_fast_static_map_fn);